My working unpac space for OCaml projects in development
Validate WEB_FEATURES.yml (#4701)
* Validate WEB_FEATURES.yml
For every pull request, verify that the `WEB_FEATURES.yml` file is
internally consistent and that the pull request does not reduce the
number of tests associated with a given classifier.
Failing for any reduction in the number of matched tests is intended to
alert contributors during file renaming operations. This failure will
need to be ignored when tests are simply deleted, and this could
potentially become a distraction for maintainers. However, test deletion
has historically been so rare that spurious failures are not expected to
be common enough to substantively interfere with routine maintenance.
* fixup! Validate WEB_FEATURES.yml
* fixup! Validate WEB_FEATURES.yml
authored by
jugglinmike
and committed by
GitHub
cd4e4319
5a630621
+413
-12
+52
vendor/git/test262/.github/workflows/checks.yml
+52
vendor/git/test262/.github/workflows/checks.yml
···
54
54
printf "::error file=%s,line=1::%s\n", substr($0, 1, i-1), substr($0, i+2);
55
55
}'
56
56
57
+
lint-web-features:
58
+
name: Lint web-features
59
+
runs-on: ubuntu-latest
60
+
steps:
61
+
- name: Checkout
62
+
uses: actions/checkout@v4
63
+
64
+
- name: Create stable copy of linting tools
65
+
run: |
66
+
cp -r tools/web-features tools/web-features-stable
67
+
68
+
- name: Install Python dependencies
69
+
run: |
70
+
python -m pip install --upgrade pip
71
+
pip install -r tools/web-features-stable/requirements.txt
72
+
73
+
- name: Validate WEB_FEATURES.yml
74
+
run: |
75
+
./tools/web-features-stable/lint.py --manifest WEB_FEATURES-manifest-pr.json
76
+
77
+
- name: Checkout base branch
78
+
uses: actions/checkout@v4
79
+
with:
80
+
ref: ${{ github.event_name == 'pull_request' && github.base_ref || github.ref_name }}
81
+
# The `clean` option must be unset in order to retain the first
82
+
# manifest file.
83
+
clean: false
84
+
85
+
- name: Generate manifest on base branch
86
+
run: |
87
+
./tools/web-features-stable/lint.py --manifest WEB_FEATURES-manifest-base.json
88
+
89
+
- name: debug
90
+
run: ls -lah
91
+
92
+
# Checking out the original branch ensures that if the patch under review
93
+
# modified `check-coverage.py` itself, that modified version is used.
94
+
- name: Checkout original branch
95
+
uses: actions/checkout@v4
96
+
with:
97
+
# The `clean` option must be unset in order to retain the manifest
98
+
# files.
99
+
clean: false
100
+
101
+
- name: debug
102
+
run: ls -lah
103
+
104
+
- name: Verify absence of regressions in classification coverage
105
+
run: |
106
+
./tools/web-features-stable/check-coverage.py \
107
+
--base WEB_FEATURES-manifest-base.json \
108
+
--new WEB_FEATURES-manifest-pr.json
57
109
58
110
build:
59
111
name: Build generated tests
+21
vendor/git/test262/.github/workflows/test-tools.yml
+21
vendor/git/test262/.github/workflows/test-tools.yml
···
50
50
51
51
- name: Test the generation tool
52
52
run: ./tools/generation/test/run.py
53
+
54
+
web-features:
55
+
name: Test the web-features tooling
56
+
runs-on: ubuntu-latest
57
+
steps:
58
+
- name: Checkout
59
+
uses: actions/checkout@v4
60
+
61
+
- name: Set up Python
62
+
uses: actions/setup-python@v5
63
+
with:
64
+
python-version: '3.x'
65
+
cache: pip
66
+
67
+
- name: Install dependencies for web-features tooling
68
+
run: |
69
+
python -m pip install --upgrade pip
70
+
pip install -r tools/web-features/requirements.txt
71
+
72
+
- name: Test the web-features tooling
73
+
run: ./tools/web-features/test/run.py
+3
vendor/git/test262/.gitignore
+3
vendor/git/test262/.gitignore
+19
-5
vendor/git/test262/tools/web-features/README.md
+19
-5
vendor/git/test262/tools/web-features/README.md
···
84
84
85
85
## Tooling
86
86
87
-
This directory defines a command-line utility in the form of an executable
88
-
Python script named `lint.py`. Upon execution, the script will validate some
89
-
basic expectations about the contents of the `WEB_FEATURES.yml` (e.g. that the
90
-
file conforms to the schema and that every pattern impacts the set of matched
91
-
test files).
87
+
This directory defines two command-line utilities in the form of executable
88
+
Python scripts. Their dependencies can be installed with the following command
89
+
(executed from the root of this repository):
90
+
91
+
pip install -r tools/web-features/requirements.txt
92
+
93
+
### `lint.py`
94
+
95
+
This script validates basic expectations about the contents of the
96
+
`WEB_FEATURES.yml` (e.g. that the file conforms to the schema and that every
97
+
pattern impacts the set of matched test files). If invoked with the argument
98
+
`--manifest`, this utility will write a "manifest" of all web-features and
99
+
their associated test file names.
100
+
101
+
### `check-coverage.py`
102
+
103
+
This script compares two web-features manifests to identify changes that likely
104
+
indicate an unintentional regression. Specifically, any reduction in the number
105
+
of tests associated with a web-feature will produce an error.
+55
vendor/git/test262/tools/web-features/check-coverage.py
+55
vendor/git/test262/tools/web-features/check-coverage.py
···
1
+
#!/usr/bin/env python3
2
+
# Copyright (C) 2025 the V8 project authors. All rights reserved.
3
+
# This code is governed by the BSD license found in the LICENSE file.
4
+
5
+
import argparse
6
+
import json
7
+
import sys
8
+
9
+
def main(base_filename, new_filename):
10
+
with open(base_filename, 'r') as base_handle:
11
+
base = json.loads(base_handle.read())
12
+
13
+
assert base['version'] == 1, 'Unsupported base manifest version'
14
+
15
+
with open(new_filename, 'r') as new_handle:
16
+
new = json.loads(new_handle.read())
17
+
18
+
assert new['version'] == 1, 'Unsupported new manifest version'
19
+
20
+
errors = []
21
+
for name, tests in base['data'].items():
22
+
assert name in new['data']
23
+
24
+
base_count = len(tests)
25
+
new_count = len(new['data'][name])
26
+
27
+
if base_count > new_count:
28
+
errors.append(
29
+
f'Web feature "{name}" appears to have regressed ' \
30
+
f'(from {base_count} matching tests to {new_count} matching ' \
31
+
f'tests). Ensure the classifiers in WEB_FEATURES.yml have ' \
32
+
f'been updated to accommodate the changes on this branch.'
33
+
)
34
+
35
+
return errors
36
+
37
+
if __name__ == '__main__':
38
+
parser = argparse.ArgumentParser(
39
+
description='Test262 web-features manifest regression checker'
40
+
)
41
+
42
+
parser.add_argument('--base', help='''A web-features manifest to use as the
43
+
basis for validation''')
44
+
parser.add_argument('--new', help='''A web-features manifest to check
45
+
for regressions''')
46
+
47
+
args = parser.parse_args()
48
+
errors = main(args.base, args.new)
49
+
50
+
if len(errors) == 0:
51
+
sys.exit(0)
52
+
else:
53
+
for error in errors:
54
+
print(error, file=sys.stderr)
55
+
sys.exit(1)
+22
-7
vendor/git/test262/tools/web-features/lint.py
+22
-7
vendor/git/test262/tools/web-features/lint.py
···
1
1
#!/usr/bin/env python3
2
2
3
+
import argparse
4
+
import json
3
5
import os
4
6
import re
7
+
import sys
5
8
import yaml
6
9
7
10
def read_features(filename):
···
25
28
return not (filename.endswith('FIXTURE.js') or filename.endswith('.json'))
26
29
27
30
def pattern_from_path_spec(path_spec):
28
-
return re.compile(re.sub('\*', '.*', path_spec) + '(/|$)')
31
+
return re.compile(re.sub('\\*', '.*', path_spec) + '(/|$)')
29
32
30
33
def get_filenames(path_spec):
31
34
pattern = pattern_from_path_spec(path_spec)
···
92
95
return False
93
96
return True
94
97
95
-
def main(web_features_filename):
98
+
def main(web_features_filename, manifest_filename):
96
99
with open(web_features_filename, 'r') as handle:
97
100
features = yaml.safe_load(handle)['features']
101
+
manifest = {'data': dict(), 'version': 1}
98
102
99
103
for feature in features:
100
104
name = feature['name']
101
105
path_specs = feature['files']
102
106
tag_specs = feature.get('tags', [])
103
107
104
-
tests = [*filter(
108
+
manifest['data'][name] = [*filter(
105
109
lambda candidate: match(candidate, tag_specs),
106
110
get_filenames_from_path_specs(path_specs)
107
111
)]
108
112
109
-
print(f'{name},{len(tests)}')
110
-
111
-
print(f'{web_features_filename} is conformant')
113
+
if manifest_filename:
114
+
with open(manifest_filename, 'w') as manifest_handle:
115
+
manifest_handle.write(json.dumps(manifest))
112
116
113
117
if __name__ == '__main__':
114
-
main('./WEB_FEATURES.yml')
118
+
parser = argparse.ArgumentParser(
119
+
description='Test262 web-features manifest regression checker'
120
+
)
121
+
122
+
parser.add_argument('--manifest', help='''The name of a JSON-formatted test
123
+
manifest file to create''')
124
+
125
+
args = parser.parse_args()
126
+
127
+
main('./WEB_FEATURES.yml', args.manifest)
128
+
129
+
print('./WEB_FEATURES.yml is conformant')
+1
vendor/git/test262/tools/web-features/requirements.txt
+1
vendor/git/test262/tools/web-features/requirements.txt
···
1
+
PyYAML==6.0.3
+240
vendor/git/test262/tools/web-features/test/run.py
+240
vendor/git/test262/tools/web-features/test/run.py
···
1
+
#!/usr/bin/env python
2
+
# Copyright (C) 2025 the V8 project authors. All rights reserved.
3
+
# This code is governed by the BSD license found in the LICENSE file.
4
+
5
+
import shutil, subprocess, sys, os, unittest, tempfile
6
+
import json
7
+
8
+
testDir = os.path.dirname(os.path.abspath(__file__))
9
+
OUT_DIR = os.path.join(testDir, 'out')
10
+
ex = os.path.join(testDir, '..', 'check-coverage.py')
11
+
12
+
class TestCheckCoverage(unittest.TestCase):
13
+
maxDiff = None
14
+
15
+
def fixture(self, name, content):
16
+
fspath = os.path.join(OUT_DIR, name)
17
+
with open(fspath, 'w') as f:
18
+
f.write(json.dumps(content))
19
+
return fspath
20
+
21
+
def check(self, args):
22
+
args[:0] = [sys.executable, ex]
23
+
sp = subprocess.Popen(args,
24
+
stdout=subprocess.PIPE,
25
+
stderr=subprocess.PIPE
26
+
)
27
+
stdout, stderr = sp.communicate()
28
+
return dict(stdout=stdout, stderr=stderr, returncode=sp.returncode)
29
+
30
+
def setUp(self):
31
+
# Defensively remove the output directory in case a critical error
32
+
# during a prior test execution prevented the removal.
33
+
shutil.rmtree(OUT_DIR, ignore_errors=True)
34
+
35
+
os.mkdir(OUT_DIR)
36
+
37
+
def tearDown(self):
38
+
shutil.rmtree(OUT_DIR, ignore_errors=True)
39
+
40
+
def test_missing_base(self):
41
+
new = self.fixture('new.json', {
42
+
'data': {},
43
+
'version': 1
44
+
})
45
+
46
+
result = self.check(['--new', new])
47
+
48
+
self.assertNotEqual(result['returncode'], 0)
49
+
50
+
def test_missing_new(self):
51
+
base = self.fixture('base.json', {
52
+
'data': {},
53
+
'version': 1
54
+
})
55
+
56
+
result = self.check(['--base', base])
57
+
58
+
self.assertNotEqual(result['returncode'], 0)
59
+
60
+
61
+
def test_empty_manifests(self):
62
+
base = self.fixture('base.json', {
63
+
'data': {},
64
+
'version': 1
65
+
})
66
+
new = self.fixture('new.json', {
67
+
'data': {},
68
+
'version': 1
69
+
})
70
+
71
+
result = self.check(['--base', base, '--new', new])
72
+
73
+
self.assertEqual(result['returncode'], 0)
74
+
75
+
def test_unsupported_base_manifest(self):
76
+
base = self.fixture('base.json', {
77
+
'data': {},
78
+
'version': 2
79
+
})
80
+
new = self.fixture('new.json', {
81
+
'data': {},
82
+
'version': 1
83
+
})
84
+
85
+
result = self.check(['--base', base, '--new', new])
86
+
87
+
self.assertNotEqual(result['returncode'], 0)
88
+
89
+
def test_unsupported_new_manifest(self):
90
+
base = self.fixture('base.json', {
91
+
'data': {},
92
+
'version': 1
93
+
})
94
+
new = self.fixture('new.json', {
95
+
'data': {},
96
+
'version': 2
97
+
})
98
+
99
+
result = self.check(['--base', base, '--new', new])
100
+
101
+
self.assertNotEqual(result['returncode'], 0)
102
+
103
+
def test_coverage_regression_removed_feature(self):
104
+
manifest_base = {
105
+
'data': {
106
+
'feature1': ['test/1-a.js'],
107
+
'feature2': ['test/2-a.js', 'test/2-b.js', 'test/2-c.js'],
108
+
'feature3': ['test/3-a.js', 'test/3-b.js']
109
+
},
110
+
'version': 1
111
+
}
112
+
manifest_new = {
113
+
'data': {
114
+
'feature1': ['test/1-a.js'],
115
+
'feature3': ['test/3-a.js', 'test/3-b.js']
116
+
},
117
+
'version': 1
118
+
}
119
+
base = self.fixture('base.json', manifest_base)
120
+
new = self.fixture('new.json', manifest_new)
121
+
122
+
result = self.check(['--base', base, '--new', new])
123
+
124
+
self.assertNotEqual(result['returncode'], 0)
125
+
126
+
def test_coverage_regression_one_fewer(self):
127
+
manifest_base = {
128
+
'data': {
129
+
'feature1': ['test/1-a.js'],
130
+
'feature2': ['test/2-a.js', 'test/2-b.js', 'test/2-c.js'],
131
+
'feature3': ['test/3-a.js', 'test/3-b.js']
132
+
},
133
+
'version': 1
134
+
}
135
+
manifest_new = {
136
+
'data': {
137
+
'feature1': ['test/1-a.js'],
138
+
'feature2': ['test/2-a.js', 'test/2-c.js'],
139
+
'feature3': ['test/3-a.js', 'test/3-b.js']
140
+
},
141
+
'version': 1
142
+
}
143
+
base = self.fixture('base.json', manifest_base)
144
+
new = self.fixture('new.json', manifest_new)
145
+
146
+
result = self.check(['--base', base, '--new', new])
147
+
148
+
self.assertNotEqual(result['returncode'], 0)
149
+
150
+
def test_identical_coverage(self):
151
+
manifest = {
152
+
'data': {
153
+
'feature1': ['test/1-a.js'],
154
+
'feature2': ['test/2-a.js', 'test/2-b.js', 'test/2-c.js'],
155
+
'feature3': ['test/3-a.js', 'test/3-b.js']
156
+
},
157
+
'version': 1
158
+
}
159
+
base = self.fixture('base.json', manifest)
160
+
new = self.fixture('new.json', manifest)
161
+
162
+
result = self.check(['--base', base, '--new', new])
163
+
164
+
self.assertEqual(result['returncode'], 0)
165
+
166
+
def test_equivalent_coverage(self):
167
+
manifest_base = {
168
+
'data': {
169
+
'feature1': ['test/1-a.js'],
170
+
'feature2': ['test/2-a.js', 'test/2-b.js', 'test/2-c.js'],
171
+
'feature3': ['test/3-a.js', 'test/3-b.js']
172
+
},
173
+
'version': 1
174
+
}
175
+
manifest_new = {
176
+
'data': {
177
+
'feature1': ['test/1-a.js'],
178
+
'feature2': ['test/2-a.js', 'test/2-b.js', 'test/2-d.js'],
179
+
'feature3': ['test/3-a.js', 'test/3-b.js']
180
+
},
181
+
'version': 1
182
+
}
183
+
base = self.fixture('base.json', manifest_base)
184
+
new = self.fixture('new.json', manifest_new)
185
+
186
+
result = self.check(['--base', base, '--new', new])
187
+
188
+
self.assertEqual(result['returncode'], 0)
189
+
190
+
def test_extended_coverage_new_file(self):
191
+
manifest_base = {
192
+
'data': {
193
+
'feature1': ['test/1-a.js'],
194
+
'feature2': ['test/2-a.js', 'test/2-b.js', 'test/2-c.js'],
195
+
'feature3': ['test/3-a.js', 'test/3-b.js']
196
+
},
197
+
'version': 1
198
+
}
199
+
manifest_new = {
200
+
'data': {
201
+
'feature1': ['test/1-a.js'],
202
+
'feature2': ['test/2-a.js', 'test/2-b.js', 'test/2-c.js'],
203
+
'feature3': ['test/3-a.js', 'test/3-b.js', 'test/3-c.js']
204
+
},
205
+
'version': 1
206
+
}
207
+
base = self.fixture('base.json', manifest_base)
208
+
new = self.fixture('new.json', manifest_new)
209
+
210
+
result = self.check(['--base', base, '--new', new])
211
+
212
+
self.assertEqual(result['returncode'], 0)
213
+
214
+
def test_extended_coverage_new_feature(self):
215
+
manifest_base = {
216
+
'data': {
217
+
'feature1': ['test/1-a.js'],
218
+
'feature2': ['test/2-a.js', 'test/2-b.js', 'test/2-c.js'],
219
+
'feature3': ['test/3-a.js', 'test/3-b.js']
220
+
},
221
+
'version': 1
222
+
}
223
+
manifest_new = {
224
+
'data': {
225
+
'feature1': ['test/1-a.js'],
226
+
'feature2': ['test/2-a.js', 'test/2-b.js', 'test/2-c.js'],
227
+
'feature3': ['test/3-a.js', 'test/3-b.js'],
228
+
'feature4': ['test/4-a.js', 'test/4-b.js']
229
+
},
230
+
'version': 1
231
+
}
232
+
base = self.fixture('base.json', manifest_base)
233
+
new = self.fixture('new.json', manifest_new)
234
+
235
+
result = self.check(['--base', base, '--new', new])
236
+
237
+
self.assertEqual(result['returncode'], 0)
238
+
239
+
if __name__ == '__main__':
240
+
unittest.main()