Import chromium-123.0.6312.40

src/net/tools/cache_transparency/checksum_pervasive_js.py

@@ -0,0 +1,40 @@
+# Copyright 2022 The Chromium Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Calculates the checksum for pervasive.js.
+
+Usage:
+  python3 checksum_pervasive_js.py pervasive.js
+
+"""
+
+import re
+import sys
+import pervasive_checksum
+
+
+def main(argv):
+  if len(argv) != 2:
+    print('Supply the path to pervasive.js as the sole command-line argument')
+    sys.exit(1)
+
+  filename = argv[1]
+  with open(filename, mode='rb') as f:
+    raw_body = f.read()
+
+  headers = []
+  with open(f'{filename}.mock-http-headers', mode='r') as lines:
+    for line in lines:
+      if line.startswith('HTTP/'):
+        continue
+      match = re.match(r'^([A-Za-z0-9-]+): *(.*)$', line)
+      if not match:
+        print(f'Failed to parse header line: {line}')
+        continue
+      headers.append((match.group(1), match.group(2)))
+
+  print(pervasive_checksum.calculate_checksum(headers, raw_body))
+
+
+if __name__ == '__main__':
+  main(sys.argv)

src/net/tools/cache_transparency/generate_checksums.py

@@ -0,0 +1,98 @@
+# Copyright 2022 The Chromium Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Generates checksums for URLs in the Pervasive Payload list."""
+
+import argparse
+import csv
+import requests
+import urllib.parse
+import pervasive_checksum
+
+
+def generate_list_with_checksums(data):
+  pairs_list = []
+  flat_list = []
+  for i, url_info in enumerate(data):
+    url = url_info[0]
+    print(f"[{i}/{len(data)}] Fetching {url}")
+
+    with requests.get(url,
+                      headers={"Accept-Encoding": "gzip, deflate, br"},
+                      stream=True) as response:
+
+      headers = list(response.headers.items())
+      raw_body = response.raw.data
+
+    checksum = pervasive_checksum.calculate_checksum(headers, raw_body)
+    pairs_list.append([url, checksum])
+    flat_list.append(str(url))
+    flat_list.append(str(checksum))
+
+  return pairs_list, flat_list
+
+
+def main():
+  parser = argparse.ArgumentParser(
+      description=__doc__, formatter_class=argparse.RawTextHelpFormatter)
+
+  parser.add_argument(
+      "input",
+      type=str,
+      nargs=1,
+      help="path for input csv file containing pervasive payloads list")
+
+  parser.add_argument("-v",
+                      "--list-version",
+                      "--version",
+                      dest="list_version",
+                      default="1",
+                      help="version of pervasive payloads list")
+
+  parser.add_argument("-f",
+                      "--format",
+                      dest="format",
+                      default="csv",
+                      choices=["csv", "comma_separated", "url_encoded"],
+                      help="output format to use. Default: csv")
+
+  parser.add_argument("output",
+                      type=str,
+                      nargs=1,
+                      help="path for output file for URLs and checksums")
+
+  args = parser.parse_args()
+
+  filename = args.input[0]
+
+  data = []
+  with open(filename, mode="r", newline="") as csvfile:
+    datareader = csv.reader(csvfile)
+    data = list(datareader)
+
+  pairs_list, flat_list = generate_list_with_checksums(data)
+
+  if args.format == "csv":
+    with open(args.output[0], mode="w", newline="") as f:
+      writer = csv.writer(f)
+      writer.writerows(pairs_list)
+
+  elif args.format == "comma_separated":
+    flat_list.insert(0, str(args.list_version))
+    with open(args.output[0], mode="w") as file:
+      file.write(",\n".join(flat_list))
+
+  elif args.format == "url_encoded":
+    concatenated = str(args.list_version) + ","
+    concatenated += ",".join(flat_list)
+    url_encoded_list = urllib.parse.quote_plus(concatenated)
+    with open(args.output[0], mode="w") as file:
+      file.write(url_encoded_list)
+    print(
+        "NOTE: To run the feature via commandline, use the following command:\n"
+        "out/Default/chrome --enable-features='PervasivePayloadsList:pervasive-payloads/(url_encoded_list),CacheTransparency,SplitCacheByNetworkIsolationKey'"
+    )
+
+
+if __name__ == "__main__":
+  main()

src/net/tools/cache_transparency/pervasive_checksum.py

@@ -0,0 +1,45 @@
+# Copyright 2022 The Chromium Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Calculates the pervasive payload checksum"""
+
+import hashlib
+
+INCLUDE_HEADERS = frozenset([
+    "access-control-allow-credentials", "access-control-allow-headers",
+    "access-control-allow-methods", "access-control-allow-origin",
+    "access-control-expose-headers", "access-control-max-age",
+    "access-control-request-headers", "access-control-request-method",
+    "clear-site-data", "content-encoding", "content-security-policy",
+    "content-type", "cross-origin-embedder-policy",
+    "cross-origin-opener-policy", "cross-origin-resource-policy", "location",
+    "sec-websocket-accept", "sec-websocket-extensions", "sec-websocket-key",
+    "sec-websocket-protocol", "sec-websocket-version", "upgrade", "vary"
+])
+
+
+def calculate_checksum(headers, raw_body):
+  """Calculates the pervasive payload checksum for a given resource
+
+  `headers` should be a list of name, value tuples.
+  `raw_body` should be the response body exactly as returned by the server,
+  without decompression or other filtering applied.
+  Returns the SHA-256 checksum of the resource, calculated per the cache
+  transparency serialization algorithm.
+  """
+
+  checksum_input = ""
+
+  headers = [(name.lower(), value) for name, value in headers]
+  headers.sort()
+
+  for header in headers:
+    if header[0] in INCLUDE_HEADERS:
+      checksum_input += header[0] + ": " + header[1] + "\n"
+
+  checksum_input += "\n"
+  checksum_input = checksum_input.encode()
+
+  checksum_input += raw_body
+
+  return hashlib.sha256(checksum_input).hexdigest().upper()

src/net/tools/cache_transparency/sample_pervasive_payloads_list.csv

@@ -0,0 +1,100 @@
+https://c.amazon-adsystem.com/aax2/apstag.js
+https://c.disquscdn.com/next/embed/common.bundle.9450cde9dd2c9d366781a8fc5ff6e933.js
+https://c.disquscdn.com/next/embed/styles/lounge.7ab903feba7624935283ca4c7d8c7203.css
+https://cdn.cookielaw.org/scripttemplates/otSDKStub.js
+https://cdn.onesignal.com/sdks/OneSignalPageSDKES6.js?v=151513
+https://cdn.onesignal.com/sdks/OneSignalSDK.js
+https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css
+https://connect.facebook.net/en_US/sdk.js
+https://connect.facebook.net/en_US/sdk.js?hash=80773c822cd6e51f7e963a4bf6a9793f
+https://connect.facebook.net/en_US/sdk.js?hash=a572daf7c4ec88ffab229a636a49a105
+https://disqus.com/next/config.js
+https://fonts.gstatic.com/s/roboto/v18/KFOmCnqEu92Fr1Mu4mxKKTU1Kg.woff2
+https://fonts.gstatic.com/s/roboto/v18/KFOmCnqEu92Fr1Mu4mxP.ttf
+https://fonts.gstatic.com/s/roboto/v29/KFOmCnqEu92Fr1Mu4mxKKTU1Kg.woff2
+https://imasdk.googleapis.com/js/sdkloader/ima3.js
+https://js-agent.newrelic.com/nr-1215.min.js
+https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css
+https://mc.yandex.com/metrika/advert.gif
+https://mc.yandex.com/sync_cookie_image_check
+https://mc.yandex.ru/metrika/advert.gif
+https://mc.yandex.ru/metrika/tag.js
+https://platform.twitter.com/embed/embed.i18n.0b77ee78a8e28a636331.js
+https://platform.twitter.com/embed/embed.i18n.293ca00a272b34d032a9.js
+https://platform.twitter.com/embed/embed.modules.aef85bf61d706d7edafa.js
+https://platform.twitter.com/embed/embed.modules.cb647a8c9f5f97575bd9.js
+https://platform.twitter.com/embed/embed.ondemand.en-js.77f81b914ed04ced0591.js
+https://platform.twitter.com/embed/embed.ondemand.en-js.e84cb370ed3e40856450.js
+https://platform.twitter.com/embed/embed.ondemand.i18n.en-js.536c09da9f27ad91baac.js
+https://platform.twitter.com/embed/embed.ondemand.i18n.en-js.5cdc09e4a37a07720433.js
+https://platform.twitter.com/embed/embed.ondemand.TimelineList~ondemand.TimelineProfile~ondemand.Tweet.44711848a6d644a51d82.js
+https://platform.twitter.com/embed/embed.ondemand.TimelineList~ondemand.TimelineProfile~ondemand.Tweet.4ec913c179dde91427eb.js
+https://platform.twitter.com/embed/embed.ondemand.Tweet.066c1f940105261b852e.js
+https://platform.twitter.com/embed/embed.ondemand.Tweet.3debca4342f31b7db9e1.js
+https://platform.twitter.com/embed/embed.ondemand.TweetVideo.40b1938448f69544d784.js
+https://platform.twitter.com/embed/embed.ondemand.TweetVideo.9677ea422e19142d19ec.js
+https://platform.twitter.com/embed/embed.runtime.1724480d574190f6b381.js
+https://platform.twitter.com/embed/embed.runtime.6b5d3661e7231f9606f3.js
+https://platform.twitter.com/embed/embed.Tweet.7e4521689aa2dd8195db.js
+https://platform.twitter.com/embed/embed.Tweet.c31baac24debe5533d7c.js
+https://platform.twitter.com/embed/embed.vendors~loaders.video.VideoPlayerDefaultUI.36474e74c34ad6f52990.js
+https://platform.twitter.com/embed/embed.vendors~loaders.video.VideoPlayerDefaultUI.96cf684c2f96172ada41.js
+https://platform.twitter.com/embed/embed.vendors~ondemand.en-js.2db7d4fd25a0403a936f.js
+https://platform.twitter.com/embed/embed.vendors~ondemand.en-js.fb959cd7154f3c56a8da.js
+https://platform.twitter.com/embed/embed.vendors~ondemand.TimelineList~ondemand.TimelineProfile~ondemand.Tweet.784c14fad3ed09aa7f26.js
+https://platform.twitter.com/embed/embed.vendors~ondemand.TimelineList~ondemand.TimelineProfile~ondemand.Tweet.8fd70d21755f2bcfae49.js
+https://platform.twitter.com/embed/embed.vendors~ondemand.TimelineList~ondemand.TimelineProfile~ondemand.TweetVideo.2690210765db59fa9244.js
+https://platform.twitter.com/embed/embed.vendors~ondemand.TimelineList~ondemand.TimelineProfile~ondemand.TweetVideo.bf89920680446a7cdc95.js
+https://platform.twitter.com/embed/embed.vendors~ondemand.Tweet.d4dc6d9f20302d27b041.js
+https://platform.twitter.com/embed/embed.vendors~ondemand.TweetVideo.98357008a9809fa238b9.js
+https://platform.twitter.com/embed/embed.vendors~ondemand.TweetVideo.c8873d7e5545d6c609c7.js
+https://platform.twitter.com/jot.html
+https://platform.twitter.com/js/button.1c2a6e168692ffea6cc8d4efc5b6f6bc.js
+https://platform.twitter.com/js/horizon_tweet.b9ac0a13a4a1d52c80651179f4fe9b68.js
+https://platform.twitter.com/js/moment~timeline.4391e0bf4053fbaa2a022e3fad2a1e1a.js
+https://platform.twitter.com/js/timeline.34cf38a85ac899f1d6a0438a1659decc.js
+https://platform.twitter.com/widgets.js
+https://s7.addthis.com/js/300/addthis_widget.js
+https://s7.addthis.com/static/layers.fa6cd1947ce26e890d3d.js
+https://s7.addthis.com/static/sh.f48a1a04fe8dbf021b4cda1d.html
+https://securepubads.g.doubleclick.net/gpt/pubads_impl_2022031601.js
+https://securepubads.g.doubleclick.net/gpt/pubads_impl_2022032101.js?cb=31065754
+https://securepubads.g.doubleclick.net/gpt/pubads_impl_2022032101.js?cb=31065822
+https://securepubads.g.doubleclick.net/gpt/pubads_impl_2022032201.js?cb=31065823
+https://snap.licdn.com/li.lms-analytics/insight.min.js
+https://ssl.google-analytics.com/ga.js
+https://static.addtoany.com/menu/page.js
+https://static.addtoany.com/menu/sm.23.html
+https://static.addtoany.com/menu/svg/icons.30.svg.js
+https://static.chartbeat.com/js/chartbeat.js
+https://static.cloudflareinsights.com/beacon.min.js/v652eace1692a40cfa3763df669d7439c1639079717194
+https://static.criteo.net/js/ld/ld.js
+https://static.doubleclick.net/instream/ad_status.js
+https://stats.wp.com/e-202212.js
+https://test.quantcast.mgr.consensu.org/GVL-v2/cmp-list.json
+https://www.google-analytics.com/analytics.js
+https://www.google-analytics.com/plugins/ua/ec.js
+https://www.google-analytics.com/plugins/ua/linkid.js
+https://www.google.com/js/th/Css0RoSI-HS3Ke9AuoQSWaa8zK9spjRs2QEDJP_UMgo.js
+https://www.google.com/js/th/eC3m08BWsKAgXdD56g8v-hRF27m9BLFWuq71lBVmRb0.js
+https://www.google.com/js/th/EplzmgSwEq1haJy_K7VkvamGBwhWugpG90UbraTSS_o.js
+https://www.google.com/js/th/GK9RXoYhzM9BS2DwszrzkGxxQbY5VR2mx9ED2vYLgv8.js
+https://www.google.com/js/th/iIdjuWWH5BicNn2_JadGpKIgqGW7lVC4-WD5EB_vw2g.js
+https://www.google.com/js/th/lgrQep_cHvWFMgJTnyMczGzKq0lFy3dFEtoDT0eWErg.js
+https://www.google.com/js/th/MKcwde2W9EpBjeUz-X8Qp_8IcWz9bCTkaBfeh4Am-J4.js
+https://www.google.com/js/th/nZvBJ6n5TdZ62aCM6LFj0MJnwO5NiOq1Qq9v_oTz3HU.js
+https://www.google.com/js/th/vUdUDDAd66Cx-AHksaT_xANRkhOhr-b_V9vr9VFFHpQ.js
+https://www.gstatic.com/recaptcha/releases/2uoiJ4hP3NUoP9v_eBNfU6CR/recaptcha__en.js
+https://www.instagram.com/embed.js
+https://www.instagram.com/static/bundles/metro/EmbedAsyncLogger.js/be2bbf5fd5e9.js
+https://www.instagram.com/static/bundles/metro/EmbedSDK.js/d9addf525b6a.js
+https://www.instagram.com/static/bundles/metro/EmbedSimple.js/0db99b108ff7.js
+https://www.instagram.com/static/bundles/metro/EmbedSimpleBase.css/d5d1c8482c44.css
+https://www.instagram.com/static/bundles/metro/en_US.js/23f9fb9aea85.js
+https://www.instagram.com/static/bundles/metro/en_US.js/2ec0b7dcc41c.js
+https://www.instagram.com/static/bundles/metro/Polyfills.js/a76474cce8e5.js
+https://www.instagram.com/static/bundles/metro/Polyfills.js/b1f149405b0e.js
+https://www.instagram.com/static/bundles/metro/Polyfills.js/fecde0d9ec0a.js
+https://www.instagram.com/static/bundles/metro/sprite_core_2x_bcd90c1d4868.png/bcd90c1d4868.png
+https://www.instagram.com/static/bundles/metro/sprite_embed_2x_644bc47b3106.png/644bc47b3106.png
+https://z.moatads.com/addthismoatframe568911941483/moatframe.js