{
  "version": 1,
  "repository": "Knuckles-Team/webarchiver",
  "revision": "a784cc5a2fd7996104ff78a201fcabcc1ffe9dd8",
  "captured_at": "2026-10-18T14:27:05.685466+00:00",
  "records": [
    {
      "id": "CONN-WEBARCHIVER-WEB-SOURCE",
      "kind": "spec",
      "parent": null,
      "source": "specs/web-source/status.json",
      "title": "CONN-WEBARCHIVER-WEB-SOURCE \u2014 Web page fetch, extraction, render and screenshot connector",
      "delivery_state": "SPECIFIED",
      "state": "specified",
      "done": false,
      "landed_in": [],
      "verified_by": [],
      "document": "specs/web-source/spec.md",
      "component": "Unassigned",
      "code_refs": [
        {
          "declared_in": "specs/web-source/plan.md",
          "evidence": "file exists; coverage not verified",
          "label": "spec.md",
          "repository": "Knuckles-Team/webarchiver",
          "role": "documentation",
          "url": "https://github.com/Knuckles-Team/webarchiver/blob/a784cc5a2fd7996104ff78a201fcabcc1ffe9dd8/specs/web-source/spec.md"
        },
        {
          "declared_in": "specs/web-source/spec.md",
          "evidence": "file exists; coverage not verified",
          "label": "requirements.md",
          "repository": "Knuckles-Team/webarchiver",
          "role": "documentation",
          "url": "https://github.com/Knuckles-Team/webarchiver/blob/a784cc5a2fd7996104ff78a201fcabcc1ffe9dd8/specs/web-source/requirements.md"
        },
        {
          "declared_in": "specs/web-source/spec.md",
          "evidence": "file exists; coverage not verified",
          "label": "test-spec.md",
          "repository": "Knuckles-Team/webarchiver",
          "role": "documentation",
          "url": "https://github.com/Knuckles-Team/webarchiver/blob/a784cc5a2fd7996104ff78a201fcabcc1ffe9dd8/specs/web-source/test-spec.md"
        }
      ],
      "relations": [
        {
          "declared_in": "specs/web-source/spec.md",
          "evidence": "external declaration; not verified",
          "kind": "extends",
          "label": "SDK-SOURCE-INGEST",
          "repository": "Knuckles-Team/agent-connector-sdk",
          "url": "https://github.com/Knuckles-Team/agent-connector-sdk/blob/main/specs/SDK-SOURCE-INGEST/spec.md"
        },
        {
          "declared_in": "specs/web-source/spec.md",
          "evidence": "external declaration; not verified",
          "kind": "references",
          "label": "ARBORA-OPS",
          "repository": "Knuckles-Team/arbora",
          "url": "https://github.com/Knuckles-Team/arbora/blob/main/specs/hosted-api-operations/spec.md"
        }
      ],
      "plan_diagrams": [],
      "unparsed_references": []
    },
    {
      "id": "CONN-WEBARCHIVER-WEB-SOURCE-R001",
      "kind": "requirement",
      "parent": "CONN-WEBARCHIVER-WEB-SOURCE",
      "source": "specs/web-source/status.json",
      "title": "Pages answer through the web-source contract (rollup)",
      "delivery_state": "SPECIFIED",
      "state": "specified",
      "done": false,
      "landed_in": [],
      "verified_by": [],
      "parent_title": "CONN-WEBARCHIVER-WEB-SOURCE \u2014 Web page fetch, extraction, render and screenshot connector",
      "definition": "**Pages answer through the web-source contract (rollup).** The webarchiver connector must implement the web-source contract of `SDK-SOURCE-INGEST-R005.40`. It serves the per-operation tools `web_fetch`, `web_extract`, `web_render` and `web_screenshot`. It is the primary backend of `research.web` (`ARBORA-OPS-R067.3`). Children `.1` to `.8` and `.10` deliver it. | See each child.",
      "definition_source": "specs/web-source/requirements.md",
      "definition_line": 9
    },
    {
      "id": "CONN-WEBARCHIVER-WEB-SOURCE-R001.1",
      "kind": "requirement",
      "parent": "CONN-WEBARCHIVER-WEB-SOURCE",
      "source": "specs/web-source/status.json",
      "title": "Model",
      "delivery_state": "VERIFIED",
      "state": "verified",
      "done": true,
      "landed_in": [
        "31e5998eddc5"
      ],
      "verified_by": [
        "tests/test_conformance.py:20"
      ],
      "parent_title": "CONN-WEBARCHIVER-WEB-SOURCE \u2014 Web page fetch, extraction, render and screenshot connector",
      "definition": "**Model.** `web_fetch` and `web_extract` are the SDK web-source contract's tools (`SDK-SOURCE-INGEST-R005.40.1`). `register_web_source_tools` serves them over `WebarchiverSource` in `webarchiver/api/web_source.py`. They take a `WebFetchRequest` and return a `WebPageResult` or a `WebFetchRefusal`, with `backend=webarchiver`. `web_render`, `web_screenshot`, `web_links` and `web_crawl` are the connector's own per-operation tools. Proving test: `tests/test_conformance.py::test_webarchiver_conforms_to_the_web_source_contract`. | The served tool schemas equal the contract's, and an unknown argument is refused.",
      "definition_source": "specs/web-source/requirements.md",
      "definition_line": 10
    },
    {
      "id": "CONN-WEBARCHIVER-WEB-SOURCE-R001.2",
      "kind": "requirement",
      "parent": "CONN-WEBARCHIVER-WEB-SOURCE",
      "source": "specs/web-source/status.json",
      "title": "Behavior",
      "delivery_state": "VERIFIED",
      "state": "verified",
      "done": true,
      "landed_in": [
        "8c38c25066e3"
      ],
      "verified_by": [
        "tests/test_extract.py:103",
        "tests/test_extract.py:117",
        "tests/test_extract.py:133",
        "tests/test_extract.py:36",
        "tests/test_extract.py:64"
      ],
      "parent_title": "CONN-WEBARCHIVER-WEB-SOURCE \u2014 Web page fetch, extraction, render and screenshot connector",
      "definition": "**Behavior.** The connector must obey `robots.txt` for its user agent and cache each file for 3600 s. It must keep a per-domain rate limit of 1 request per second by default. It must stop reading at `max_bytes` and refuse an unlisted content type. `web_extract` must reduce the page to its main content as markdown. It must return the title, author, published date, canonical URL and outbound links. `content_digest` is the SHA-256 of the extracted text. Proving test: `tests/test_extract.py::test_main_content_and_metadata`. | Fixture pages yield the expected markdown and metadata, and a disallowed path is refused.",
      "definition_source": "specs/web-source/requirements.md",
      "definition_line": 11
    },
    {
      "id": "CONN-WEBARCHIVER-WEB-SOURCE-R001.3",
      "kind": "requirement",
      "parent": "CONN-WEBARCHIVER-WEB-SOURCE",
      "source": "specs/web-source/status.json",
      "title": "Wiring",
      "delivery_state": "VERIFIED",
      "state": "verified",
      "done": true,
      "landed_in": [
        "8c38c25066e3"
      ],
      "verified_by": [
        "tests/test_served.py:34",
        "tests/test_served.py:52",
        "tests/test_served.py:69"
      ],
      "parent_title": "CONN-WEBARCHIVER-WEB-SOURCE \u2014 Web page fetch, extraction, render and screenshot connector",
      "definition": "**Wiring.** The browser capture in `Webarchiver.launch_browser` (`webarchiver/webarchiver.py:63`) serves `web_render` and `web_screenshot`. The server uses the SDK server factory and `register_tool_surface`. The production caller is `WebFetchSourceAdapter.extract` of `SDK-SOURCE-INGEST-R005.40.3`. Proving test: `tests/test_served.py::test_tools_call_reaches_extractor`. | A served `tools/call` reaches the extractor and the capture path.",
      "definition_source": "specs/web-source/requirements.md",
      "definition_line": 12
    },
    {
      "id": "CONN-WEBARCHIVER-WEB-SOURCE-R001.4",
      "kind": "requirement",
      "parent": "CONN-WEBARCHIVER-WEB-SOURCE",
      "source": "specs/web-source/status.json",
      "title": "Refusal",
      "delivery_state": "VERIFIED",
      "state": "verified",
      "done": true,
      "landed_in": [
        "8c38c25066e3"
      ],
      "verified_by": [
        "tests/test_extract.py:45"
      ],
      "parent_title": "CONN-WEBARCHIVER-WEB-SOURCE \u2014 Web page fetch, extraction, render and screenshot connector",
      "definition": "**Refusal.** Each refusal must use a closed reason of the contract: `robots_disallowed`, `rate_limited`, `size_exceeded`, `unsupported_type`, `egress_denied`, `policy_refused`, `extract_empty` or `upstream_failed`. A private address must refuse through the SDK egress policy. The connector must never return an empty page as a success. Proving test: `tests/test_extract.py::test_refusals`. | Each case returns its reason and no page body.",
      "definition_source": "specs/web-source/requirements.md",
      "definition_line": 13
    },
    {
      "id": "CONN-WEBARCHIVER-WEB-SOURCE-R001.5",
      "kind": "requirement",
      "parent": "CONN-WEBARCHIVER-WEB-SOURCE",
      "source": "specs/web-source/status.json",
      "title": "Evidence",
      "delivery_state": "VERIFIED",
      "state": "verified",
      "done": true,
      "landed_in": [
        "8c38c25066e3"
      ],
      "verified_by": [
        "tests/test_extract.py:143"
      ],
      "parent_title": "CONN-WEBARCHIVER-WEB-SOURCE \u2014 Web page fetch, extraction, render and screenshot connector",
      "definition": "**Evidence.** Each result must name `backend=webarchiver`, the final URL, the HTTP status, the byte count, the content digest, the retrieval time and the `robots.txt` decision. Proving test: `tests/test_extract.py::test_result_provenance`. | Each named field is present and matches the fixture response.",
      "definition_source": "specs/web-source/requirements.md",
      "definition_line": 14
    },
    {
      "id": "CONN-WEBARCHIVER-WEB-SOURCE-R001.6",
      "kind": "requirement",
      "parent": "CONN-WEBARCHIVER-WEB-SOURCE",
      "source": "specs/web-source/status.json",
      "title": "Hot-path proof",
      "delivery_state": "SPECIFIED",
      "state": "specified",
      "done": false,
      "landed_in": [],
      "verified_by": [],
      "parent_title": "CONN-WEBARCHIVER-WEB-SOURCE \u2014 Web page fetch, extraction, render and screenshot connector",
      "definition": "**Hot-path proof.** The test serves the connector and runs the SDK conformance kit of `SDK-SOURCE-INGEST-R005.40.2` against it. It then runs `connector-sync --once` for one page into a running engine. Proving test: `tests/test_conformance_served.py::test_conformance_kit_and_sync`. | The kit reports no broken obligation, and the engine holds one Document with `backend=webarchiver`.",
      "definition_source": "specs/web-source/requirements.md",
      "definition_line": 15
    },
    {
      "id": "CONN-WEBARCHIVER-WEB-SOURCE-R001.7",
      "kind": "requirement",
      "parent": "CONN-WEBARCHIVER-WEB-SOURCE",
      "source": "specs/web-source/status.json",
      "title": "Bounded site crawl",
      "delivery_state": "VERIFIED",
      "state": "verified",
      "done": true,
      "landed_in": [
        "4fde2c6fe305",
        "99c1281925fe"
      ],
      "verified_by": [
        "tests/test_crawl.py:17",
        "tests/test_crawl.py:35",
        "tests/test_crawl.py:49"
      ],
      "parent_title": "CONN-WEBARCHIVER-WEB-SOURCE \u2014 Web page fetch, extraction, render and screenshot connector",
      "definition": "**Bounded site crawl.** `web_crawl(seed_url, same_site, max_pages, max_depth, include, exclude)` in `webarchiver/api/crawl.py` must walk links breadth-first from one seed and return the page URLs it reached, in discovery order. It must fetch every page through the same policy as `web_fetch`: robots.txt, the per-domain rate limit, size and content-type caps and egress pinning. `same_site` defaults to true. `max_pages` ranges from 1 to 1000 and `max_depth` from 0 to 10. `include` and `exclude` are regular expressions. A robots-disallowed or failed page is reported as skipped with its closed reason. It returns URLs only; fetching content is the caller's next step. Proving test: `tests/test_crawl.py::test_bounded_crawl`. | A fixture site yields the expected URLs in order, the disallowed path is reported and never fetched, and out-of-range bounds refuse with `policy_refused`.",
      "definition_source": "specs/web-source/requirements.md",
      "definition_line": 16
    },
    {
      "id": "CONN-WEBARCHIVER-WEB-SOURCE-R001.8",
      "kind": "requirement",
      "parent": "CONN-WEBARCHIVER-WEB-SOURCE",
      "source": "specs/web-source/status.json",
      "title": "Ingest mappings",
      "delivery_state": "VERIFIED",
      "state": "verified",
      "done": true,
      "landed_in": [
        "482ceea819b0",
        "4d127d07b8ce"
      ],
      "verified_by": [
        "tests/test_startup.py:51"
      ],
      "parent_title": "CONN-WEBARCHIVER-WEB-SOURCE \u2014 Web page fetch, extraction, render and screenshot connector",
      "definition": "**Ingest mappings.** `webarchiver/connector_manifest.yml` must declare `schema_mappings` for the `Document` and `Chunk` record types, mapped to the core `Document` and `Chunk` classes, so `SourceIngest` resolves `manifest:<connector>#schema_mappings/Document` and `.../Chunk` in the pack the fleet imports (`SDK-SOURCE-INGEST-R004.25`). Proving test: `tests/test_startup.py::test_manifest_maps_document_and_chunk_records`. | The served manifest maps both record types with their text, title, source and backend fields.",
      "definition_source": "specs/web-source/requirements.md",
      "definition_line": 17
    },
    {
      "id": "CONN-WEBARCHIVER-WEB-SOURCE-R001.10",
      "kind": "requirement",
      "parent": "CONN-WEBARCHIVER-WEB-SOURCE",
      "source": "specs/web-source/status.json",
      "title": "Conformance",
      "delivery_state": "VERIFIED",
      "state": "verified",
      "done": true,
      "landed_in": [
        "31e5998eddc5"
      ],
      "verified_by": [
        "tests/test_conformance.py:21"
      ],
      "parent_title": "CONN-WEBARCHIVER-WEB-SOURCE \u2014 Web page fetch, extraction, render and screenshot connector",
      "definition": "**Conformance.** The connector passes the SDK conformance kit (`SDK-SOURCE-INGEST-R005.40.2`) against itself, served in process. The kit checks robots.txt, the per-domain rate limit, `max_bytes`, the content type, the digest and the tool schemas. The archivebox-api connector passes the same kit, so both produce the same tools and record shape (`CONN-ARCHIVEBOX-API-WEB-SOURCE-R001.8`). Proving test: `tests/test_conformance.py::test_webarchiver_conforms_to_the_web_source_contract`. | The kit reports no broken obligation.",
      "definition_source": "specs/web-source/requirements.md",
      "definition_line": 18
    }
  ],
  "github": {
    "issues": {
      "available": true,
      "items": []
    },
    "pull_requests": {
      "available": true,
      "items": []
    }
  },
  "spec_edges": [
    [
      "CONN-WEBARCHIVER-WEB-SOURCE",
      "extends",
      "https://github.com/Knuckles-Team/agent-connector-sdk/blob/main/specs/SDK-SOURCE-INGEST/spec.md"
    ],
    [
      "CONN-WEBARCHIVER-WEB-SOURCE",
      "references",
      "https://github.com/Knuckles-Team/arbora/blob/main/specs/hosted-api-operations/spec.md"
    ]
  ]
}