import json,sys,urllib.request,concurrent.futures as cf
f=sys.argv[1]; d=json.load(open(f)); rows=[r for r in d['rows'] if r['status']=='ok']
def chk(r):
    base=r['url'].rsplit('/robots.txt',1)[0]
    try:
        req=urllib.request.Request(base+'/llms.txt',headers={"User-Agent":"Mozilla/5.0 (compatible; OarsmanResearch/1.0; llms.txt study)"})
        x=urllib.request.urlopen(req,timeout=12); t=x.read(200000).decode('utf-8','replace'); ct=x.headers.get('Content-Type','')
        if '<html' in t[:800].lower() or '<!doctype' in t[:200].lower(): return r['domain'],'html'
        h1=any(l.startswith('# ') for l in t.splitlines()[:20])
        return r['domain'],'valid-h1' if h1 else 'text-no-h1'
    except urllib.error.HTTPError as e: return r['domain'],f'http{e.code}'
    except Exception: return r['domain'],'err'
from collections import Counter
with cf.ThreadPoolExecutor(32) as ex: res=list(ex.map(chk,rows))
c=Counter(s for _,s in res); print(f,len(rows),c)
json.dump(res,open(f.replace('robots-study','llms-study'),'w'))
print([d for d,s in res if s=='valid-h1'][:40])