Add PyTest support for Mac, remove pylint & dependency validation
Take care of a few repository maintenance and cleanup tasks:
- Add support to run pytest on Mac: google-crc32c was not available for the mac platform in version 1.1.2, but support was added in 1.3.0
- Use YAPF instead of pylint: YAPF was already suggested to format text, but applies different rules than pylint which was used in presubmit. YAPF is now used for lint checking in presubmit as well.
- Validate generated dependencies: When running `tools/generate_requirements_txt.py`, new recursive dependencies might be added which are not defined in .vpython3. This is an issue for running tests (which rely on cipd wheels defined in .vpython3), and a security concern due to the inclusion of unreviewed versions. This change validates that all (recursive) dependencies added to requirements.txt have been defined in .vpython3 and provides helpful messages if validation fails, e.g.
```
Exception: Generated requirements.txt differs from definitions in .vpython3:
· charset-normalizer==2.0.4 found in requirements.txt, but .vpython3 defines version 1.1.0
· pyasn1-modules==0.2.8 found in requirements.txt, but not in .vpython3
Fix the .vpython3 definitions and re-run this command.
```
- Add tmrts & machenbach as OWNERS
Change-Id: I587f2df3211ef56241892d47d089375b4d79ae9d
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/chrome-devtools-frontend/+/3691840
Auto-Submit: Alexander Schulze <alexschulze@chromium.org>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
diff --git a/OWNERS b/OWNERS
index e56520c..fce38cc 100644
--- a/OWNERS
+++ b/OWNERS
@@ -4,6 +4,8 @@
yangguo@chromium.org
alexschulze@chromium.org
liviurau@chromium.org
+machenbach@chromium.org
+tmrts@chromium.org
# Whitespace file to test the continuous infrastructure.
per-file whitespace.txt=*
diff --git a/gae_py3/.vpython3 b/gae_py3/.vpython3
index 7347c18..b094b78 100644
--- a/gae_py3/.vpython3
+++ b/gae_py3/.vpython3
@@ -116,7 +116,7 @@
wheel: <
name: "infra/python/wheels/google-crc32c/${vpython_platform}"
- version: "version:1.1.2"
+ version: "version:1.3.0"
>
wheel: <
@@ -232,3 +232,19 @@
name: "infra/python/wheels/markdown-py3"
version: "version:3.3.4"
>
+
+wheel: <
+ name: "infra/python/wheels/importlib-metadata-py2_py3"
+ version: "version:1.6.0"
+>
+
+wheel: <
+ name: "infra/python/wheels/zipp-py3"
+ version: "version:3.7.0"
+>
+
+# YAPF
+wheel: <
+ name: "infra/python/wheels/yapf-py3"
+ version: "version:0.32.0"
+>
diff --git a/gae_py3/PRESUBMIT.py b/gae_py3/PRESUBMIT.py
index e9dd007..779fa3b 100644
--- a/gae_py3/PRESUBMIT.py
+++ b/gae_py3/PRESUBMIT.py
@@ -8,32 +8,43 @@
USE_PYTHON3 = True
-def CommonChecks(input_api, output_api, pylint_only_changed=True):
- results = []
+def _RunYapf(input_api, output_api, yapf_only_changed=True):
+ presubmit_base_path = input_api.PresubmitLocalPath()
- # Run Pylint over the files in the directory.
- if pylint_only_changed:
- # Get relative paths (to this PRESUBMIT.py) for all changed files
+ if input_api.is_committing:
+ error_message = output_api.PresubmitError
+ else:
+ error_message = output_api.PresubmitPromptWarning
+
+ cmd = ['yapf', '--diff']
+ if yapf_only_changed:
changed_files = input_api.change.AbsoluteLocalPaths()
changed_py_files = filter(lambda f: f.endswith('.py'), changed_files)
- presubmit_base_path = input_api.PresubmitLocalPath()
- pylint_files_to_check = [
+ cmd += [
input_api.os_path.relpath(cf, presubmit_base_path)
for cf in changed_py_files
]
else:
- pylint_files_to_check = None
+ cmd += ['--recursive', presubmit_base_path]
- # GetPylint runs on all files if an empty list is provided for
- # `files_to_check`, but no files got changed
- if pylint_files_to_check is None or len(pylint_files_to_check) > 0:
- results.extend(
- input_api.RunTests(
- input_api.canned_checks.GetPylint(
- input_api,
- output_api,
- files_to_check=pylint_files_to_check,
- version='2.6')))
+ return input_api.RunTests([
+ input_api.Command(
+ name='Running YAPF; call `yapf -irp .` to fix formatting issues',
+ cmd=cmd,
+ message=error_message,
+ kwargs={},
+ python3=True,
+ )
+ ])
+
+
+def CommonChecks(input_api, output_api, yapf_only_changed=True):
+ results = []
+
+ # Run YAPF
+ results.extend(_RunYapf(input_api, output_api, yapf_only_changed))
+
+ return results
# Run Python unittests.
results.extend(
@@ -47,8 +58,8 @@
def CheckChangeOnUpload(input_api, output_api):
- return CommonChecks(input_api, output_api, pylint_only_changed=True)
+ return CommonChecks(input_api, output_api, yapf_only_changed=True)
def CheckChangeOnCommit(input_api, output_api):
- return CommonChecks(input_api, output_api, pylint_only_changed=False)
+ return CommonChecks(input_api, output_api, yapf_only_changed=False)
diff --git a/gae_py3/config.py b/gae_py3/config.py
index 4785eae..28dc457 100644
--- a/gae_py3/config.py
+++ b/gae_py3/config.py
@@ -12,11 +12,11 @@
CHROME_UNSIGNED_BUCKET = "chrome-unsigned"
CHROME_UNSIGNED_ARTIFACT_PATH = "desktop-5c0tCh/%s/mac64/devtools-frontend.zip"
CHROME_UNSIGNED_ZIP_BASE_DIRS = [
- "devtools-frontend/gen/third_party/devtools-frontend-internal/devtools-frontend/front_end/", # pylint: disable=C0301
- "devtools-frontend/gen/third_party/devtools-frontend-internal-ng/devtools-frontend/front_end/", # pylint: disable=C0301
+ "devtools-frontend/gen/third_party/devtools-frontend-internal/devtools-frontend/front_end/",
+ "devtools-frontend/gen/third_party/devtools-frontend-internal-ng/devtools-frontend/front_end/",
]
LEGACY_BUCKET = "chrome-devtools-frontend"
LOCAL_BUCKET = os.getenv("LOCAL_BUCKET")
-IS_GAE = os.getenv("GAE_ENV") == "standard"
\ No newline at end of file
+IS_GAE = os.getenv("GAE_ENV") == "standard"
diff --git a/gae_py3/requirements.txt b/gae_py3/requirements.txt
index 2789d3f..a9157aa 100644
--- a/gae_py3/requirements.txt
+++ b/gae_py3/requirements.txt
@@ -13,57 +13,6 @@
certifi==2021.5.30 \
--hash=sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee \
--hash=sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8
-cffi==1.15.0 \
- --hash=sha256:00c878c90cb53ccfaae6b8bc18ad05d2036553e6d9d1d9dbcf323bbe83854ca3 \
- --hash=sha256:0104fb5ae2391d46a4cb082abdd5c69ea4eab79d8d44eaaf79f1b1fd806ee4c2 \
- --hash=sha256:06c48159c1abed75c2e721b1715c379fa3200c7784271b3c46df01383b593636 \
- --hash=sha256:0808014eb713677ec1292301ea4c81ad277b6cdf2fdd90fd540af98c0b101d20 \
- --hash=sha256:10dffb601ccfb65262a27233ac273d552ddc4d8ae1bf93b21c94b8511bffe728 \
- --hash=sha256:14cd121ea63ecdae71efa69c15c5543a4b5fbcd0bbe2aad864baca0063cecf27 \
- --hash=sha256:17771976e82e9f94976180f76468546834d22a7cc404b17c22df2a2c81db0c66 \
- --hash=sha256:181dee03b1170ff1969489acf1c26533710231c58f95534e3edac87fff06c443 \
- --hash=sha256:23cfe892bd5dd8941608f93348c0737e369e51c100d03718f108bf1add7bd6d0 \
- --hash=sha256:263cc3d821c4ab2213cbe8cd8b355a7f72a8324577dc865ef98487c1aeee2bc7 \
- --hash=sha256:2756c88cbb94231c7a147402476be2c4df2f6078099a6f4a480d239a8817ae39 \
- --hash=sha256:27c219baf94952ae9d50ec19651a687b826792055353d07648a5695413e0c605 \
- --hash=sha256:2a23af14f408d53d5e6cd4e3d9a24ff9e05906ad574822a10563efcef137979a \
- --hash=sha256:31fb708d9d7c3f49a60f04cf5b119aeefe5644daba1cd2a0fe389b674fd1de37 \
- --hash=sha256:3415c89f9204ee60cd09b235810be700e993e343a408693e80ce7f6a40108029 \
- --hash=sha256:3773c4d81e6e818df2efbc7dd77325ca0dcb688116050fb2b3011218eda36139 \
- --hash=sha256:3b96a311ac60a3f6be21d2572e46ce67f09abcf4d09344c49274eb9e0bf345fc \
- --hash=sha256:3f7d084648d77af029acb79a0ff49a0ad7e9d09057a9bf46596dac9514dc07df \
- --hash=sha256:41d45de54cd277a7878919867c0f08b0cf817605e4eb94093e7516505d3c8d14 \
- --hash=sha256:4238e6dab5d6a8ba812de994bbb0a79bddbdf80994e4ce802b6f6f3142fcc880 \
- --hash=sha256:45db3a33139e9c8f7c09234b5784a5e33d31fd6907800b316decad50af323ff2 \
- --hash=sha256:45e8636704eacc432a206ac7345a5d3d2c62d95a507ec70d62f23cd91770482a \
- --hash=sha256:4958391dbd6249d7ad855b9ca88fae690783a6be9e86df65865058ed81fc860e \
- --hash=sha256:4a306fa632e8f0928956a41fa8e1d6243c71e7eb59ffbd165fc0b41e316b2474 \
- --hash=sha256:57e9ac9ccc3101fac9d6014fba037473e4358ef4e89f8e181f8951a2c0162024 \
- --hash=sha256:59888172256cac5629e60e72e86598027aca6bf01fa2465bdb676d37636573e8 \
- --hash=sha256:5e069f72d497312b24fcc02073d70cb989045d1c91cbd53979366077959933e0 \
- --hash=sha256:64d4ec9f448dfe041705426000cc13e34e6e5bb13736e9fd62e34a0b0c41566e \
- --hash=sha256:6dc2737a3674b3e344847c8686cf29e500584ccad76204efea14f451d4cc669a \
- --hash=sha256:74fdfdbfdc48d3f47148976f49fab3251e550a8720bebc99bf1483f5bfb5db3e \
- --hash=sha256:75e4024375654472cc27e91cbe9eaa08567f7fbdf822638be2814ce059f58032 \
- --hash=sha256:786902fb9ba7433aae840e0ed609f45c7bcd4e225ebb9c753aa39725bb3e6ad6 \
- --hash=sha256:8b6c2ea03845c9f501ed1313e78de148cd3f6cad741a75d43a29b43da27f2e1e \
- --hash=sha256:91d77d2a782be4274da750752bb1650a97bfd8f291022b379bb8e01c66b4e96b \
- --hash=sha256:91ec59c33514b7c7559a6acda53bbfe1b283949c34fe7440bcf917f96ac0723e \
- --hash=sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954 \
- --hash=sha256:a5263e363c27b653a90078143adb3d076c1a748ec9ecc78ea2fb916f9b861962 \
- --hash=sha256:abb9a20a72ac4e0fdb50dae135ba5e77880518e742077ced47eb1499e29a443c \
- --hash=sha256:c2051981a968d7de9dd2d7b87bcb9c939c74a34626a6e2f8181455dd49ed69e4 \
- --hash=sha256:c21c9e3896c23007803a875460fb786118f0cdd4434359577ea25eb556e34c55 \
- --hash=sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962 \
- --hash=sha256:d4d692a89c5cf08a8557fdeb329b82e7bf609aadfaed6c0d79f5a449a3c7c023 \
- --hash=sha256:da5db4e883f1ce37f55c667e5c0de439df76ac4cb55964655906306918e7363c \
- --hash=sha256:e7022a66d9b55e93e1a845d8c9eba2a1bebd4966cd8bfc25d9cd07d515b33fa6 \
- --hash=sha256:ef1f279350da2c586a69d32fc8733092fd32cc8ac95139a00377841f59a3f8d8 \
- --hash=sha256:f54a64f8b0c8ff0b64d18aa76675262e1700f3995182267998c31ae974fbc382 \
- --hash=sha256:f5c7150ad32ba43a07c4479f40241756145a1f03b43480e058cfd862bf5041c7 \
- --hash=sha256:f6f824dc3bce0edab5f427efcfb1d63ee75b6fcb7282900ccaf925be84efb0fc \
- --hash=sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997 \
- --hash=sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796
chardet==4.0.0 \
--hash=sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa \
--hash=sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5
@@ -97,36 +46,50 @@
google-cloud-storage==2.1.0 \
--hash=sha256:0a5e7ab1a38d2c24be8e566e50b8b0daa8af8fd49d4ab312b1fda5c147429893 \
--hash=sha256:53e4f6030d771526e4dbe9c6000cca44eab812f39ae8c7a810be999473e4f02c
-google-crc32c==1.1.2 \
- --hash=sha256:0ae3cf54e0d4d83c8af1afe96fc0970fbf32f1b29275f3bfd44ce25c4b622a2b \
- --hash=sha256:0dd9b61d0c63043b013349c9ec8a83ec2b05c96410c5bc257da5d0de743fc171 \
- --hash=sha256:110157fb19ab5db15603debfaf5fcfbac9627576787d9caf8618ff96821a7a1f \
- --hash=sha256:1dc6904c0d958f43102c85d70792cca210d3d051ddbeecd0eff10abcd981fdfa \
- --hash=sha256:298a9a922d35b123a73be80233d0f19c6ea01f008743561a8937f9dd83fb586b \
- --hash=sha256:34a97937f164147aefa53c3277364fd3bfa7fd244cbebbd5a976fa8325fb496b \
- --hash=sha256:364eb36e8d9d34542c17b0c410035b0557edd4300a92ed736b237afaa0fd6dae \
- --hash=sha256:49838ede42592154f9fcd21d07c7a43a67b00a36e252f82ae72542fde09dc51f \
- --hash=sha256:51f4aa06125bf0641f65fb83268853545dbeb36b98ccfec69ef57dcb6b73b176 \
- --hash=sha256:6789db0b12aab12a0f04de22ed8412dfa5f6abd5a342ea19f15355064e1cc387 \
- --hash=sha256:78cf5b1bd30f3a6033b41aa4ce8c796870bc4645a15d3ef47a4b05d31b0a6dc1 \
- --hash=sha256:7c5138ed2e815189ba524756e027ac5833365e86115b1c2e6d9e833974a58d82 \
- --hash=sha256:80abca603187093ea089cd1215c3779040dda55d3cdabc0cd5ea0e10df7bff99 \
- --hash=sha256:8ed8f6dc4f55850cba2eb22b78902ad37f397ee02692d3b8e00842e9af757321 \
- --hash=sha256:91ad96ee2958311d0bb75ffe5c25c87fb521ef547c09e04a8bb6143e75fb1367 \
- --hash=sha256:92ed6062792b989e84621e07a5f3d37da9cc3153b77d23a582921f14863af31d \
- --hash=sha256:9372211acbcc207f63ffaffea1d05f3244a21311e4710721ffff3e8b7a0d24d0 \
- --hash=sha256:a64e0e8ed6076a8d867fc4622ad821c55eba8dff1b48b18f56b7c2392e22ab9d \
- --hash=sha256:a6c8a712ffae56c805ca732b735af02860b246bed2c1acb38ea954a8b2dc4581 \
- --hash=sha256:ab2b31395fbeeae6d15c98bd7f8b9fb76a18f18f87adc11b1f6dbe8f90d8382f \
- --hash=sha256:ae7b9e7e2ca1b06c3a68b6ef223947a52c30ffae329b1a2be3402756073f2732 \
- --hash=sha256:b5ea1055fe470334ced844270e7c808b04fe31e3e6394675daa77f6789ca9eff \
- --hash=sha256:d0630670d27785d7e610e72752dc8087436d00d2c7115e149c0a754babb56d3e \
- --hash=sha256:d4a0d4fb938c2c3c0076445c9bd1215a3bd3df557b88d8b05ec2889ca0c92f8d \
- --hash=sha256:dff5bd1236737f66950999d25de7a78144548ebac7788d30ada8c1b6ead60b27 \
- --hash=sha256:e5af77656e8d367701f40f80a91c985ca43319f322f0a36ba9f93909d0bc4cb2 \
- --hash=sha256:e6458c41236d37cb982120b070ebcc115687c852bee24cdd18792da2640cf44d \
- --hash=sha256:ea170341a4a9078a067b431044cd56c73553425833a7c2bb81734777a230ad4b \
- --hash=sha256:ef2ed6d0ac4de4ac602903e203eccd25ec8e37f1446fe1a3d2953a658035e0a5
+google-crc32c==1.3.0 \
+ --hash=sha256:04e7c220798a72fd0f08242bc8d7a05986b2a08a0573396187fd32c1dcdd58b3 \
+ --hash=sha256:05340b60bf05b574159e9bd940152a47d38af3fb43803ffe71f11d704b7696a6 \
+ --hash=sha256:12674a4c3b56b706153a358eaa1018c4137a5a04635b92b4652440d3d7386206 \
+ --hash=sha256:127f9cc3ac41b6a859bd9dc4321097b1a4f6aa7fdf71b4f9227b9e3ebffb4422 \
+ --hash=sha256:13af315c3a0eec8bb8b8d80b8b128cb3fcd17d7e4edafc39647846345a3f003a \
+ --hash=sha256:1926fd8de0acb9d15ee757175ce7242e235482a783cd4ec711cc999fc103c24e \
+ --hash=sha256:226f2f9b8e128a6ca6a9af9b9e8384f7b53a801907425c9a292553a3a7218ce0 \
+ --hash=sha256:276de6273eb074a35bc598f8efbc00c7869c5cf2e29c90748fccc8c898c244df \
+ --hash=sha256:318f73f5484b5671f0c7f5f63741ab020a599504ed81d209b5c7129ee4667407 \
+ --hash=sha256:3bbce1be3687bbfebe29abdb7631b83e6b25da3f4e1856a1611eb21854b689ea \
+ --hash=sha256:42ae4781333e331a1743445931b08ebdad73e188fd554259e772556fc4937c48 \
+ --hash=sha256:58be56ae0529c664cc04a9c76e68bb92b091e0194d6e3c50bea7e0f266f73713 \
+ --hash=sha256:5da2c81575cc3ccf05d9830f9e8d3c70954819ca9a63828210498c0774fda1a3 \
+ --hash=sha256:6311853aa2bba4064d0c28ca54e7b50c4d48e3de04f6770f6c60ebda1e975267 \
+ --hash=sha256:650e2917660e696041ab3dcd7abac160b4121cd9a484c08406f24c5964099829 \
+ --hash=sha256:6a4db36f9721fdf391646685ecffa404eb986cbe007a3289499020daf72e88a2 \
+ --hash=sha256:779cbf1ce375b96111db98fca913c1f5ec11b1d870e529b1dc7354b2681a8c3a \
+ --hash=sha256:7f6fe42536d9dcd3e2ffb9d3053f5d05221ae3bbcefbe472bdf2c71c793e3183 \
+ --hash=sha256:891f712ce54e0d631370e1f4997b3f182f3368179198efc30d477c75d1f44942 \
+ --hash=sha256:95c68a4b9b7828ba0428f8f7e3109c5d476ca44996ed9a5f8aac6269296e2d59 \
+ --hash=sha256:96a8918a78d5d64e07c8ea4ed2bc44354e3f93f46a4866a40e8db934e4c0d74b \
+ --hash=sha256:9c3cf890c3c0ecfe1510a452a165431b5831e24160c5fcf2071f0f85ca5a47cd \
+ --hash=sha256:9f58099ad7affc0754ae42e6d87443299f15d739b0ce03c76f515153a5cda06c \
+ --hash=sha256:a0b9e622c3b2b8d0ce32f77eba617ab0d6768b82836391e4f8f9e2074582bf02 \
+ --hash=sha256:a7f9cbea4245ee36190f85fe1814e2d7b1e5f2186381b082f5d59f99b7f11328 \
+ --hash=sha256:bab4aebd525218bab4ee615786c4581952eadc16b1ff031813a2fd51f0cc7b08 \
+ --hash=sha256:c124b8c8779bf2d35d9b721e52d4adb41c9bfbde45e6a3f25f0820caa9aba73f \
+ --hash=sha256:c9da0a39b53d2fab3e5467329ed50e951eb91386e9d0d5b12daf593973c3b168 \
+ --hash=sha256:ca60076c388728d3b6ac3846842474f4250c91efbfe5afa872d3ffd69dd4b318 \
+ --hash=sha256:cb6994fff247987c66a8a4e550ef374671c2b82e3c0d2115e689d21e511a652d \
+ --hash=sha256:d1c1d6236feab51200272d79b3d3e0f12cf2cbb12b208c835b175a21efdb0a73 \
+ --hash=sha256:dd7760a88a8d3d705ff562aa93f8445ead54f58fd482e4f9e2bafb7e177375d4 \
+ --hash=sha256:dda4d8a3bb0b50f540f6ff4b6033f3a74e8bf0bd5320b70fab2c03e512a62812 \
+ --hash=sha256:e0f1ff55dde0ebcfbef027edc21f71c205845585fffe30d4ec4979416613e9b3 \
+ --hash=sha256:e7a539b9be7b9c00f11ef16b55486141bc2cdb0c54762f84e3c6fc091917436d \
+ --hash=sha256:eb0b14523758e37802f27b7f8cd973f5f3d33be7613952c0df904b68c4842f0e \
+ --hash=sha256:ed447680ff21c14aaceb6a9f99a5f639f583ccfe4ce1a5e1d48eb41c3d6b3217 \
+ --hash=sha256:f52a4ad2568314ee713715b1e2d79ab55fab11e8b304fd1462ff5cccf4264b3e \
+ --hash=sha256:fbd60c6aaa07c31d7754edbc2334aef50601b7f1ada67a96eb1eb57c7c72378f \
+ --hash=sha256:fc28e0db232c62ca0c3600884933178f0825c99be4474cdd645e378a10588125 \
+ --hash=sha256:fe31de3002e7b08eb20823b3735b97c86c5926dd0581c7710a680b418a8709d4 \
+ --hash=sha256:fec221a051150eeddfdfcff162e6db92c65ecf46cb0f7bb1bf812a1520ec026b \
+ --hash=sha256:ff71073ebf0e42258a42a0b34f2c09ec384977e7f6808999102eedd5b49920e3
google-resumable-media==2.3.0 \
--hash=sha256:1a7dce5790b04518edc02c2ce33965556660d64957106d66a945086e2b642572 \
--hash=sha256:36dc2f7201ee1cb360ef502187aa4e1f2b6ec4467fcee92e08a8cf165e36f587
@@ -186,6 +149,9 @@
idna==2.10 \
--hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \
--hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0
+importlib-metadata==1.6.0 \
+ --hash=sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f \
+ --hash=sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e
iniconfig==1.1.1 \
--hash=sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3 \
--hash=sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32
@@ -313,9 +279,6 @@
pyasn1-modules==0.2.8 \
--hash=sha256:905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e \
--hash=sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74
-pycparser==2.21 \
- --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \
- --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206
pyparsing==2.4.7 \
--hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \
--hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b
@@ -343,3 +306,9 @@
werkzeug==2.0.1 \
--hash=sha256:1de1db30d010ff1af14a009224ec49ab2329ad2cde454c8a708130642d579c42 \
--hash=sha256:6c1ec500dcdba0baa27600f6a22f6333d8b662d22027ff9f6202e3367413caa8
+yapf==0.32.0 \
+ --hash=sha256:8fea849025584e486fd06d6ba2bed717f396080fd3cc236ba10cb97c4c51cf32 \
+ --hash=sha256:a3f5085d37ef7e3e004c4ba9f9b3e40c54ff1901cd111f05145ae313a7c67d1b
+zipp==3.7.0 \
+ --hash=sha256:9f50f446828eb9d45b267433fd3e9da8d801f614129124863f9c51ebceafb87d \
+ --hash=sha256:b47250dd24f92b7dd6a0a8fc5244da14608f3ca90a5efcd37a3b1642fac9a375
diff --git a/gae_py3/tools/generate_requirements_txt.py b/gae_py3/tools/generate_requirements_txt.py
index 43d7c22..9684561 100755
--- a/gae_py3/tools/generate_requirements_txt.py
+++ b/gae_py3/tools/generate_requirements_txt.py
@@ -61,7 +61,9 @@
name = rm_prefix(cipd_name, 'infra/python/wheels/')
name = name.rsplit('/', 1)[0] # Remove trailing `/${vpython_platform}`
name = rm_suffix(name, '-py2_py3')
- return rm_suffix(name, '-py3')
+ name = rm_suffix(name, '-py3')
+ name = name.replace('_', '-')
+ return name
def parse(self) -> List[Tuple[str, str]]:
"""Read the .vpython3 file and create a list of pinned dependencies."""
@@ -83,10 +85,13 @@
return results
-def generate_requirements_in():
- seen = set()
+def retrieve_deps() -> List[Tuple[str, str]]:
vpython_deps = BASE_PATH.joinpath('.vpython3')
- deps = MANUAL_PACKAGES + VPythonParser(vpython_deps).parse()
+ return MANUAL_PACKAGES + VPythonParser(vpython_deps).parse()
+
+
+def generate_requirements_in(deps: List[Tuple[str, str]]):
+ seen = set()
# Only add first appearance of each package
cleaned_deps = list()
@@ -130,12 +135,46 @@
txt.write(message)
+def validate_recursive_dependencies(defined_deps: List[Tuple[str, str]]):
+ """Assert that all recursive dependencies are defined in .vpython3 too."""
+ generated_deps = []
+ with open(REQUIREMENTS_TXT) as f:
+ for line in filter(lambda line: '==' in line, f.readlines()):
+ name, version = line.split('==')
+ version = version[:-3]
+ generated_deps.append((name, version))
+
+ defined_deps_by_name = dict(defined_deps)
+
+ errors = []
+ for name, version in generated_deps:
+ if name not in defined_deps_by_name:
+ errors.append(
+ f'{name}=={version} found in requirements.txt, but not in .vpython3')
+ continue
+ if defined_deps_by_name[name] != version:
+ errors.append(
+ f'{name}=={version} found in requirements.txt, but .vpython3 defines '
+ f'version {defined_deps_by_name[name]}')
+
+ if len(errors) > 0:
+ # Add error formatting and provide hints to fix the issues:
+ errors_msg = '\n'.join([' · ' + error for error in errors])
+ raise Exception(
+ 'Generated requirements.txt differs from definitions in .vpython3:\n\n'
+ + errors_msg + '\n\n' +
+ 'Fix the .vpython3 definitions and re-run this command.')
+
+
def cleanup():
os.remove(REQUIREMENTS_IN)
if __name__ == '__main__':
- generate_requirements_in()
+ deps = retrieve_deps()
+
+ generate_requirements_in(deps)
compile_requirements_txt()
+ validate_recursive_dependencies(deps)
add_auto_generation_disclaimer()
cleanup()
