import sys, json, csv, concurrent.futures as cf, urllib.request, urllib.error, datetime
sys.path.insert(0,'/workspace/dhan/products/claude-skill-packs/sampler/oarsman-skills-sampler/skills/ai-crawler-check/scripts')
import ai_robots_check as a
N=int(sys.argv[1]) if len(sys.argv)>1 else 1000
S=int(sys.argv[2]) if len(sys.argv)>2 else 0
doms=[r[1] for r in csv.reader(open('top-1m.csv'))][S:S+N]
def fetch(d):
    for u in (f"https://{d}/robots.txt", f"https://www.{d}/robots.txt"):
        try:
            req=urllib.request.Request(u,headers={"User-Agent":"Mozilla/5.0 (compatible; OarsmanResearch/1.0; robots.txt study)"})
            r=urllib.request.urlopen(req,timeout=12)
            ct=r.headers.get('Content-Type','')
            t=r.read(500000).decode('utf-8','replace')
            if '<html' in t[:500].lower(): return d,u,'html',None
            return d,u,'ok',t
        except urllib.error.HTTPError as e:
            if 400<=e.code<500: return d,u,f'http{e.code}',''
            last=f'http{e.code}'
        except Exception as e: last='err'
    return d,None,last,None
rows=[]
with cf.ThreadPoolExecutor(32) as ex:
    for d,u,st,t in ex.map(fetch,doms):
        row={"domain":d,"status":st,"url":u}
        if t is not None:
            g=a.parse(t)
            row["bots"]={b:a.allowed(a.rules_for(g,b),"/") for b in a.BOTS}
            row["mentions"]={b:(b.lower() in t.lower()) for b in a.BOTS}
            row["has_llms_hint"]=('llms.txt' in t.lower())
        rows.append(row)
json.dump({"list":"Tranco top-1m, list id 746VX, downloaded 2026-11-01 IST","n":N,"fetched_at":datetime.datetime.now().isoformat(),"rows":rows},open(f'robots-study-s{S}-n{N}-2026-11-01.json','w'),indent=0)
ok=[r for r in rows if 'bots' in r]
print("domains",N,"robots readable (incl 4xx=allow-all)",len(ok))
from collections import Counter
print(Counter(r['status'] for r in rows))
for b in a.BOTS:
    bl=sum(1 for r in ok if not r['bots'][b]); print(f"{b:18} blocked at / : {bl}/{len(ok)} = {100*bl/len(ok):.1f}%  named explicitly: {sum(1 for r in ok if r['mentions'][b])}")
g_only=sum(1 for r in ok if not r['bots']['GPTBot'] and r['bots']['OAI-SearchBot'])
both=sum(1 for r in ok if not r['bots']['GPTBot'] and not r['bots']['OAI-SearchBot'])
print("block GPTBot but allow OAI-SearchBot",g_only,"; block both",both)
print("block ClaudeBot but allow Claude-SearchBot",sum(1 for r in ok if not r['bots']['ClaudeBot'] and r['bots']['Claude-SearchBot']))
print("block GPTBot but allow ChatGPT-User",sum(1 for r in ok if not r['bots']['GPTBot'] and r['bots']['ChatGPT-User']))
print("Googlebot blocked",sum(1 for r in ok if not r['bots']['Googlebot']))
print("any AI bot blocked",sum(1 for r in ok if any(not r['bots'][b] for b in ['GPTBot','OAI-SearchBot','ClaudeBot','Claude-SearchBot','PerplexityBot','Google-Extended','CCBot'])))
print("search-type AI bot blocked (OAI-SearchBot/Claude-SearchBot/PerplexityBot)",sum(1 for r in ok if any(not r['bots'][b] for b in ['OAI-SearchBot','Claude-SearchBot','PerplexityBot'])))
print("robots.txt mentions llms.txt",sum(1 for r in ok if r.get('has_llms_hint')))