タグ: 生成AI

import json
from pathlib import Path

URL = 'https://example.com/guide/'
OTHER = 'https://example.com/original/'
CANON = f'<link rel="canonical" href="{URL}">'
XML = f'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><url><loc>{URL}</loc></url></urlset>'

def fixture(name, meta='', canonical=CANON, body='<h1>手順</h1>', want=True,
            preferred=URL, robots='User-agent: *\nDisallow:\n', sitemap=XML, fragment=False, base=''):
    d = Path('fixtures') / name
    d.mkdir(parents=True, exist_ok=True)
    html = body if fragment else f'<!doctype html><html lang="ja"><head><meta charset="utf-8"><title>手順</title>{base}{meta}{canonical}</head><body>{body}</body></html>'
    for file, text in [('page.html', html), ('robots.txt', robots), ('sitemap.xml', sitemap),
                       ('intent.json', json.dumps({'url': URL, 'want_index': want, 'preferred': preferred}, ensure_ascii=False, indent=2))]:
        (d / file).write_text(text, encoding='utf-8')

fixture('normal')
fixture('noindex', meta='<meta name="robots" content="noindex, follow">')
fixture('fragment', body='<h1>本文だけ</h1>', fragment=True)
fixture('conflict', canonical=CANON + f'<link rel="canonical" href="{OTHER}">')
fixture('intended', meta='<meta name="robots" content="noindex">', want=False)
fixture('unknown-intent', meta='<meta name="robots" content="noindex">', want=None)
fixture('googlebot-none', meta='<meta name="googlebot" content="NONE">')
fixture('cross-canonical', canonical=f'<link rel="canonical" href="{OTHER}">', preferred=OTHER,
        sitemap=XML.replace(URL, OTHER))
fixture('missing-canonical', canonical='')
fixture('blocked-noindex', meta='<meta name="robots" content="noindex">', robots='User-agent: *\nDisallow: /guide/\n')
fixture('complex-robots', robots='User-agent: *\nDisallow: /\nAllow: /guide/\n')
fixture('broken-sitemap', sitemap='<urlset>')
fixture('two-h1', body='<h1>手順</h1><h1>別の区分</h1>',
        meta="""<!-- <meta name="robots" content="noindex"> --><script>const s = '<meta name="robots" content="noindex">';</script>""")
fixture('sitemap-index', sitemap='<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"></sitemapindex>')
fixture('base-relative', canonical='<link rel="canonical" href="guide/">', base='<base href="https://example.com/">')
fixture('relative', canonical='<link rel="canonical" href="/guide/">')
fixture('body-noindex', body='<meta name="robots" content="noindex"><h1>手順</h1>')
# WordPress標準に近いrobots.txtとコアのsitemap index（この最小コードでは判定不能になる例）
WP_ROBOTS = 'User-agent: *\nDisallow: /wp-admin/\nAllow: /wp-admin/admin-ajax.php\n\nSitemap: https://example.com/wp-sitemap.xml\n'
WP_INDEX = '<?xml version="1.0" encoding="UTF-8"?><sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><sitemap><loc>https://example.com/wp-sitemap-posts-post-1.xml</loc></sitemap></sitemapindex>'
fixture('wp-default', robots=WP_ROBOTS, sitemap=WP_INDEX)

import json
import re
import sys
from html.parser import HTMLParser
from pathlib import Path
from urllib.parse import urljoin, urlsplit
import xml.etree.ElementTree as ET

P, N, U = '問題候補', '範囲内で正常', '判定不能'

class Page(HTMLParser):
    def __init__(self):
        super().__init__()
        self.head = False
        self.events, self.robots, self.canon = [], [], []
        self.base, self.h1 = False, 0

    def handle_starttag(self, tag, attrs):
        a = dict(attrs)
        if tag in ('html', 'head', 'body'):
            self.events.append(tag)
        if tag == 'head':
            self.head = True
        if tag == 'h1':
            self.h1 += 1
        if tag == 'meta' and (a.get('name') or '').lower() in ('robots', 'googlebot'):
            self.robots += re.split(r'[\s,]+', (a.get('content') or '').lower())
        if not self.head:
            return
        if tag == 'base':
            self.base = True
        if tag == 'link' and 'canonical' in (a.get('rel') or '').lower().split():
            self.canon.append(a.get('href') or '')

    def handle_endtag(self, tag):
        if tag in ('html', 'head', 'body'):
            self.events.append('/' + tag)
        if tag == 'head':
            self.head = False


def audit(folder):
    d = Path(folder)
    cfg = json.loads((d / 'intent.json').read_text(encoding='utf-8'))
    url, want, preferred = cfg['url'], cfg['want_index'], cfg['preferred']
    h = Page()
    h.feed((d / 'page.html').read_text(encoding='utf-8'))
    full = h.events == ['html', 'head', '/head', 'body', '/body', '/html']
    out = {k: U for k in ('meta', 'canonical', 'robots', 'sitemap', 'http', 'index', 'cwv')}
    noindex = bool({'noindex', 'none'} & set(h.robots))
    if full:
        out['meta'] = U if want is None else (P if noindex and want else N)
        targets = {urljoin(url, x) for x in h.canon if x}
        if not h.base and h.canon:
            if any(not x for x in h.canon) or len(targets) > 1:
                out['canonical'] = P
            elif preferred and all(urlsplit(x).scheme in ('http', 'https') for x in targets):
                out['canonical'] = N if targets == {preferred} else P
    # 教材専用: 単一の User-agent: * と単純な Disallow 接頭辞だけ扱う。
    lines = [(x.split('#', 1)[0]).strip() for x in (d / 'robots.txt').read_text(encoding='utf-8').splitlines()]
    lines = [x for x in lines if x]
    rules, supported = [], bool(lines) and lines[0].lower() == 'user-agent: *'
    for line in lines[1:]:
        key, sep, value = line.partition(':')
        value = value.strip()
        if not sep or key.lower() != 'disallow' or (value and not value.startswith('/')) or any(c in value for c in '*$%?'):
            supported = False
        rules.append(value)
    if supported:
        blocked = any(x and urlsplit(url).path.startswith(x) for x in rules)
        out['robots'] = (U if want is None else P if want else N) if blocked else N
    try:
        root = ET.fromstring((d / 'sitemap.xml').read_text(encoding='utf-8'))
        ns = '{http://www.sitemaps.org/schemas/sitemap/0.9}'
        if root.tag == ns + 'urlset':
            locs = [x.text for x in root.findall(ns + 'url/' + ns + 'loc')]
            out['sitemap'] = N if preferred and preferred in locs else U
    except ET.ParseError:
        out['sitemap'] = P
    return {'checks': out, 'observed': {'complete_markers': full, 'html_noindex': noindex if full else None,
            'canonical_hrefs': h.canon if full else None, 'h1_count': h.h1},
            'scope': '確認したのは保存したHTML・robots.txt・sitemap.xml。正常は各項目の確認範囲に限る。HTTP・登録状態・CWVは別に取得して確かめる'}

if __name__ == '__main__':
    print(json.dumps(audit(sys.argv[1]), ensure_ascii=False, indent=2))

import json
import runpy
from pathlib import Path
from audit import audit, P, N, U

runpy.run_path('fixtures.py')
# 期待値：例ごとに、記事の3分類（問題候補／範囲内で正常／判定不能）で決めた値。
expected = {
    'normal': [N,N,N,N], 'noindex': [P,N,N,N], 'fragment': [U,U,N,N],
    'conflict': [N,P,N,N], 'intended': [N,N,N,N], 'unknown-intent': [U,N,N,N],
    'googlebot-none': [P,N,N,N], 'cross-canonical': [N,N,N,N],
    'missing-canonical': [N,U,N,N], 'blocked-noindex': [P,N,P,N],
    'complex-robots': [N,N,U,N], 'broken-sitemap': [N,N,N,P],
    'two-h1': [N,N,N,N], 'sitemap-index': [N,N,N,U], 'base-relative': [N,U,N,N],
    'relative': [N,N,N,N], 'body-noindex': [P,N,N,N], 'wp-default': [N,N,U,U],
}
results = {}
for name, values in expected.items():
    result = audit(Path('fixtures') / name)
    assert [result['checks'][k] for k in ('meta','canonical','robots','sitemap')] == values, name
    assert all(result['checks'][k] == U for k in ('http','index','cwv')), name
    results[name] = result
assert results['two-h1']['observed']['h1_count'] == 2
assert results['fragment']['observed']['html_noindex'] is None
assert results['two-h1']['observed']['html_noindex'] is False
Path('results.json').write_text(json.dumps(results, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
Path('checks.json').write_text(json.dumps({'fixture_cases':len(expected),'status_assertions':len(expected)*7,
    'observation_assertions':3,'passed':True,'expected':expected}, ensure_ascii=False, indent=2)+'\n', encoding='utf-8')
print(f'CHECK_OK: {len(expected)} cases, {len(expected)*7} statuses, 3 observations')