python-sdk/.github/workflows/conformance.yml at main · modelcontextprotocol/python-sdk · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
name: Conformance Tests

on:
  push:
    branches: [main]
  pull_request:
  workflow_dispatch:

concurrency:
  group: conformance-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

env:
  # Pinned conformance harness package spec (passed verbatim to `npx --yes`).
  # Use a published version, e.g. @modelcontextprotocol/conformance@0.2.0-alpha.7.
  # Bump deliberately and reconcile both
  # .github/actions/conformance/expected-failures*.yml files in the same change.
  #
  # Temporarily pinned to the pkg.pr.new build of conformance main@4944b268
  # (0.2.0-alpha.8, which includes #372: fail checks whose prerequisite is
  # missing instead of skipping them) — alpha.8 is not published to npm yet.
  # Pinned by commit SHA so the tarball cannot move under us;
  # CONFORMANCE_PKG_SHA256 pins the bytes and the fetch-and-verify step below
  # downloads, checks the digest, and repoints CONFORMANCE_PKG at the
  # verified local copy. Repin to the next published @modelcontextprotocol/
  # conformance release (>=0.2.0-alpha.8) once it ships, then drop
  # CONFORMANCE_PKG_SHA256 and the fetch-and-verify steps.
  CONFORMANCE_PKG: "https://pkg.pr.new/@modelcontextprotocol/conformance@4944b268"
  CONFORMANCE_PKG_SHA256: "0f70c035782d319d72ab427653c5275db5c50429d59fae0241a645b33aeda1a7"

jobs:
  server-conformance:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          persist-credentials: false
      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
        with:
          enable-cache: true
          version: 0.9.5
      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version: 24
      - name: Fetch and verify conformance harness
        # Only when CONFORMANCE_PKG is a URL: download, check the recorded
        # sha256, and re-point CONFORMANCE_PKG at the verified local tarball.
        # When CONFORMANCE_PKG is a registry spec, this step is a no-op (npm's
        # own integrity check applies).
        run: |
          case "$CONFORMANCE_PKG" in
            https://*)
              curl -fsSL "$CONFORMANCE_PKG" -o /tmp/conformance.tgz
              echo "$CONFORMANCE_PKG_SHA256  /tmp/conformance.tgz" | sha256sum -c -
              echo "CONFORMANCE_PKG=file:/tmp/conformance.tgz" >> "$GITHUB_ENV"
              ;;
          esac
      - run: uv sync --frozen --all-extras --package mcp-everything-server
      - name: Run server conformance (active suite)
        run: >-
          ./.github/actions/conformance/run-server.sh
          --suite active
          --expected-failures ./.github/actions/conformance/expected-failures.yml
      - name: Run server conformance (draft suite)
        run: >-
          ./.github/actions/conformance/run-server.sh
          --suite draft
          --expected-failures ./.github/actions/conformance/expected-failures.yml
      - name: Run server conformance (2026-07-28 wire, all suite)
        run: >-
          ./.github/actions/conformance/run-server.sh
          --suite all
          --spec-version 2026-07-28
          --expected-failures ./.github/actions/conformance/expected-failures.2026-07-28.yml
      - name: Run server conformance (all suite, extension scenarios)
        # A bare `--suite all` (no --spec-version) selects every scenario
        # shipped with the pinned harness — including the extension-tagged
        # tasks-* scenarios and pending-listed ones like server-sse-polling,
        # which no other leg reaches (extension scenarios never match a
        # --spec-version filter, and the pending list keeps them out of the
        # active suite). Running the full set keeps unimplemented surfaces
        # visible as baselined known failures in expected-failures.yml instead
        # of silent exclusions, and stays robust to scenarios moving between
        # harness suite lists across pin bumps. `--suite pending` would cover
        # the same union slightly faster; the full set is preferred for the
        # self-contained run and for parity with typescript-sdk's CI.
        run: >-
          ./.github/actions/conformance/run-server.sh
          --suite all
          --expected-failures ./.github/actions/conformance/expected-failures.yml

  client-conformance:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          persist-credentials: false
      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
        with:
          enable-cache: true
          version: 0.9.5
      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version: 24
      - name: Fetch and verify conformance harness
        # Only when CONFORMANCE_PKG is a URL: download, check the recorded
        # sha256, and re-point CONFORMANCE_PKG at the verified local tarball.
        # When CONFORMANCE_PKG is a registry spec, this step is a no-op (npm's
        # own integrity check applies).
        run: |
          case "$CONFORMANCE_PKG" in
            https://*)
              curl -fsSL "$CONFORMANCE_PKG" -o /tmp/conformance.tgz
              echo "$CONFORMANCE_PKG_SHA256  /tmp/conformance.tgz" | sha256sum -c -
              echo "CONFORMANCE_PKG=file:/tmp/conformance.tgz" >> "$GITHUB_ENV"
              ;;
          esac
      - run: uv sync --frozen --all-extras --package mcp
      - name: Run client conformance (all suite)
        # The harness runs all scenarios via unbounded Promise.all; with 40
        # scenarios on a 2-core runner the slowest one (sse-retry, which has a
        # real-time SSE reconnect wait) needs more than the 30s default budget.
        run: >-
          npx --yes "$CONFORMANCE_PKG" client
          --command 'uv run --frozen python .github/actions/conformance/client.py'
          --suite all
          --timeout 60000
          --expected-failures ./.github/actions/conformance/expected-failures.yml
      - name: Run client conformance (2026-07-28 wire, all suite)
        run: >-
          npx --yes "$CONFORMANCE_PKG" client
          --command 'uv run --frozen python .github/actions/conformance/client.py'
          --suite all
          --timeout 60000
          --spec-version 2026-07-28
          --expected-failures ./.github/actions/conformance/expected-failures.2026-07-28.yml