Files
digital-psychology/scrape_bb.py
T

45 lines
1.7 KiB
Python

import json, subprocess, os
BASE='https://minke8.cn'
# Scales that need tmnr_bb=1
bb_scales = [
("xl2","明尼苏达多项人格测验(MMPI)","心理测试综合", 1),
("xl3","艾森克人格问卷(EPQ)","人格测试", 1),
("xl7","康奈尔医学指数(CMI)","心理测试综合", 1),
]
results=[]
for slug,name,cat,bb in bb_scales:
url=f'{BASE}/{slug}.html'
print(f'{name} (bb={bb})', flush=True)
try:
r=subprocess.run(f'curl -sL -X POST {url} -d "action=process&index=0&tmnr_bb={bb}"', shell=True, capture_output=True, text=True, timeout=15)
d=json.loads(r.stdout)
qc=d.get('length',0)
if qc==0: print(f' SKIP'); continue
except: print(f' FAIL'); continue
qs=[]; os_=[]
for i in range(qc):
try:
r=subprocess.run(f'curl -sL -X POST {url} -d "action=process&index={i}&tmnr_bb={bb}"', shell=True, capture_output=True, text=True, timeout=10)
d=json.loads(r.stdout)
qs.append(d.get('tm',''))
os_.append(d.get('xx',[]))
except: qs.append(''); os_.append([])
if i%30==29: print('.', end='', flush=True)
results.append({'slug':slug,'name':name,'category':cat,'url':url,'description':'','question_count':qc,'questions':qs,'options':os_})
print(f' OK({qc})')
# Merge
existing=[]
if os.path.exists('scales_data.json'):
with open('scales_data.json') as f: existing=json.load(f)
existing_slugs={s['slug'] for s in existing}
for r in results:
if r['slug'] not in existing_slugs:
existing.append(r)
with open('scales_data.json','w') as f: json.dump(existing, f, ensure_ascii=False, indent=2)
total_q=sum(s['question_count'] for s in existing)
print(f'Saved: {len(existing)} scales, {total_q} questions')