sapling getdeps cli build and test support for python 3.12

Summary:
Ubuntu 24.04 and Fedora 40 are both python 3.12 based.  Update sapling's OSS getdeps build to allow build/run/test on python 3.12 while keeping 3.10 support.

* distutils is deprecated in python 3.12 stdlib, but fortunately setuptools is pretty similar and includes a vendored distutils, so updated to use setuptools's version and removed deprecated usage like `find_executable` in favor of `shutil.which()`.
* added a manifest so we get python3-setuptools installed.
* found some tests needed bunzip2, so update manifest for sapling and bz2 to install it
* util.py makedate():  datetime.utcfromtimestamp is deprecated in 3.12, so updated to use non-deprecated methods..
 Tested with test-command-template.t which shows tz offset.
* Makefile: disabled a few tests on 3.12 where updating the expectation to run on 3.12 and 3.10 is tricky
* tests modified to run on 3.12 and 3.10:
 * test-import-eol.t:  updated for invalid escape sequence warnings
 * test-install.t: updated to filter out message that appears in different order on 3.12 vs earlier versions
 * test-sign-commit.t: updated to add --yes for newer gpg that python 3.12 using distros have

Test Plan:

enter ubuntu 24.04 toolbox:
`toolbox enter ubuntu-toolbox-24.04 `

make sure system packages are installed
`./build/fbcode_builder/getdeps.py install-system-deps --recursive  sapling`

check oss non-thrift build:
```
make oss
$ ./sl --version
Fri Oct  4 17:14:19 BST 2024
Sapling 4.4.2_20241004_170203_06bd09bd1dd2
(see https://sapling-scm.com/ for more information)
```

build sapling getdepts
`./build/fbcode_builder/getdeps.py build --allow-system-packages --no-facebook-internal --src-dir=. sapling  --project-install-prefix sapling:/`

run the tests locally to validate python 3.12 operation:
```
./build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. sapling --num-jobs=64 --project-install-prefix sapling:/'
```

generate github actions CI to pick up python3-setuptools
```
./build/fbcode_builder/getdeps.py --allow-system-packages generate-github-actions --free-up-disk --os-type=linux --src-dir=. --output-dir=.github/workflows --job-name "Sapling CLI Getdeps " --job-file-prefix=sapling-cli-getdeps_ sapling --num-jobs=16  --project-install-prefix sapling:/
```

github CI

.github/workflows/sapling-cli-getdeps_linux.yml

@@ -72,6 +72,8 @@ jobs:
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests snappy
  - name: Fetch zlib
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests zlib
+ - name: Fetch python-setuptools
+ run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests python-setuptools
  - name: Fetch openssl
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests openssl
  - name: Fetch liboqs
@@ -146,6 +148,8 @@ jobs:
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests snappy
  - name: Build zlib
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests zlib
+ - name: Build python-setuptools
+ run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests python-setuptools
  - name: Build openssl
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests openssl
  - name: Build liboqs

build/fbcode_builder/manifests/bz2

@@ -3,12 +3,14 @@ name = bz2
 
 [debs]
 libbz2-dev
+bzip2
 
 [homebrew]
 bzip2
 
 [rpms]
 bzip2-devel
+bzip2
 
 [download]
 url = https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz

build/fbcode_builder/manifests/python-setuptools

@@ -7,3 +7,12 @@ sha256 = 02fa291a0471b3a18b2b2481ed902af520c69e8ae0919c13da936542754b4c56
 
 [build]
 builder = python-wheel
+
+[rpms]
+python3-setuptools
+
+[homebrew]
+python-setuptools
+
+[debs]
+python3-setuptools

build/fbcode_builder/manifests/sapling

@@ -65,6 +65,7 @@ hexdump
 
 [dependencies.not(os=windows)]
 python
+python-setuptools
 
 # We use the system openssl on linux
 [dependencies.not(os=linux)]

eden/integration/lib/find_executables.py

@@ -11,7 +11,7 @@
 
 # pyre-strict
 
-import distutils.spawn
+import shutil
 import logging
 import os
 import sys
@@ -192,9 +192,9 @@ def DROP_PRIVS(self) -> str:
 
  @cached_property
  def GIT(self) -> str:
- git = distutils.spawn.find_executable(
+ git = shutil.which(
  "git.real"
- ) or distutils.spawn.find_executable("git")
+ ) or shutil.which("git")
  if git is None:
  raise Exception("unable to find git binary")
  return git
@@ -224,7 +224,7 @@ def _find_hg(self) -> str:
  if hg_bin:
  return hg_bin
 
- hg_real_bin = distutils.spawn.find_executable("hg")
+ hg_real_bin = shutil.which("hg")
  if hg_real_bin:
  return hg_real_bin
 
@@ -238,11 +238,11 @@ def _find_hg_real(self) -> str:
  if hg_real_bin:
  return hg_real_bin
 
- hg_real_bin = distutils.spawn.find_executable("hg.real")
+ hg_real_bin = shutil.which("hg.real")
  if hg_real_bin:
  return hg_real_bin
 
- hg_real_bin = distutils.spawn.find_executable("hg")
+ hg_real_bin = shutil.which("hg")
  if hg_real_bin:
  return hg_real_bin
 

eden/scm/Makefile

@@ -218,6 +218,14 @@ GETDEPS_TEST_EXCLUSION_LIST := test-cats.t \
  test-rust-checkout.t \
  test-smartlog-interactive.t \
  test-smartlog-interactive-highlighting.t
+
+ifeq ($(PYTHON_MINOR_VERSION),12)
+# output differs for these on 3.12 in ways that are hard to match for 3.10 and 3.12 simultaneously
+GETDEPS_TEST_EXCLUSION_LIST := $(GETDEPS_TEST_EXCLUSION_LIST) \
+ test-eager-exchange.t \
+ test-git-bundle.t \
+ test-mergedriver2.t
+endif
 
 # convert to a sed expression
 GETDEPS_TEST_EXCLUSIONS := $(subst $() $(),|,$(GETDEPS_TEST_EXCLUSION_LIST))

eden/scm/contrib/check-code.py

@@ -415,7 +415,7 @@ def rephere(m):
 
 txtpats = [
  [
- ("\s$", "trailing whitespace"),
+ ("\\s$", "trailing whitespace"),
  (".. note::[ \n][^\n]", "add two newlines after note::"),
  ],
  [],

eden/scm/contrib/fix-code.py

@@ -21,6 +21,7 @@
 import os
 import re
 import sys
+import setuptools
 from distutils.version import LooseVersion
 
 

eden/scm/distutils_rust/__init__.py

@@ -7,14 +7,18 @@
 
 from __future__ import absolute_import
 
+import logging
+log = logging.getLogger()
+
 import contextlib
-import distutils
-import distutils.command.build
-import distutils.command.install
-import distutils.command.install_scripts
-import distutils.core
-import distutils.errors
-import distutils.util
+from setuptools import Command
+from distutils.command.build import build
+from distutils.command.install import install
+from distutils.command.install_scripts import install_scripts
+import setuptools.errors
+from distutils.dist import Distribution
+
+from sysconfig import get_platform
 import errno
 import io
 import os
@@ -79,7 +83,7 @@ def __init__(
 
  @property
  def dstnametmp(self):
- platform = distutils.util.get_platform()
+ platform = get_platform()
  if platform.startswith("win"):
  name = self.name + ".dll"
  elif platform.startswith("macosx"):
@@ -90,7 +94,7 @@ def dstnametmp(self):
 
  @property
  def dstname(self):
- platform = distutils.util.get_platform()
+ platform = get_platform()
  if platform.startswith("win"):
  name = self.name + ".pyd"
  else:
@@ -128,22 +132,22 @@ def __init__(
 
  @property
  def dstnametmp(self):
- platform = distutils.util.get_platform()
+ platform = get_platform()
  if platform.startswith("win"):
  return self.name + ".exe"
  else:
  return self.name
 
  @property
  def dstname(self):
- platform = distutils.util.get_platform()
+ platform = get_platform()
  if platform.startswith("win"):
  return self.final_name + ".exe"
  else:
  return self.final_name
 
 
-class BuildRustExt(distutils.core.Command):
+class BuildRustExt(Command):
 
  description = "build Rust extensions (compile/link to build directory)"
 
@@ -230,7 +234,7 @@ def get_output_filename(self, target):
  elif target.type == "binary":
  return os.path.join(self.build_exe, target.dstname)
  else:
- raise distutils.errors.CompileError("Unknown Rust target type")
+ raise setuptools.errors.CompileError("Unknown Rust target type")
 
  def write_cargo_config(self):
  config = """
@@ -336,15 +340,15 @@ def build_target(self, target):
 
  rc = _callretry(cmd, env=env)
  if rc:
- raise distutils.errors.CompileError(
+ raise setuptools.errors.CompileError(
  "compilation of Rust target '%s' failed" % target.name
  )
 
  src = os.path.join(self.get_cargo_target(), self.get_temp_output(target))
 
  if (
  target.type == "binary"
- and distutils.util.get_platform().startswith("win")
+ and get_platform().startswith("win")
  and self.long_paths_support
  ):
  retry = 0
@@ -358,7 +362,7 @@ def build_target(self, target):
  if retry > 5:
  raise
  else:
- distutils.log.warn("Retrying setting long path on %s" % src)
+ log.warning("Retrying setting long path on %s" % src)
  continue
  else:
  break
@@ -379,17 +383,17 @@ def build_target(self, target):
  shutil.copy(pdbsrc, pdbdest)
 
  def set_long_paths_manifest(self, fname):
- if not distutils.util.get_platform().startswith("win"):
+ if not get_platform().startswith("win"):
  # This only makes sense on Windows
- distutils.log.info(
+ log.info(
  "skipping set_long_paths_manifest call for %s "
  "as the plaform in not Windows",
  fname,
  )
  return
  if not fname.endswith(".exe"):
  # we only care about executables
- distutils.log.info(
+ log.info(
  "skipping set_long_paths_manifest call for %s "
  "as the file extension is not exe",
  fname,
@@ -400,7 +404,7 @@ def set_long_paths_manifest(self, fname):
  os.close(fdauto)
  with open(manfname, "w") as f:
  f.write(LONG_PATHS_MANIFEST)
- distutils.log.debug(
+ log.debug(
  "LongPathsAware manifest written into tempfile: %s", manfname
  )
  inputresource = "-inputresource:%s;#1" % fname
@@ -414,23 +418,23 @@ def set_long_paths_manifest(self, fname):
  inputresource,
  ]
  try:
- distutils.log.debug(
+ log.debug(
  "Trying to merge LongPathsAware manifest with the existing "
  "manifest of the binary %s",
  fname,
  )
  subprocess.check_output(command)
- distutils.log.debug(
+ log.debug(
  "LongPathsAware manifest successfully merged into %s", fname
  )
  except subprocess.CalledProcessError as e:
  no_resource_err = b"c101008c"
  if no_resource_err not in e.output.lower():
- distutils.log.error(
+ log.error(
  "Setting LongPathsAware manifest failed: %r", e.output
  )
  raise
- distutils.log.debug(
+ log.debug(
  "The binary %s does not have an existing manifest. Writing "
  "a LongPathsAware one",
  fname,
@@ -439,18 +443,18 @@ def set_long_paths_manifest(self, fname):
  # but rather just write one.
  command.remove(inputresource)
  subprocess.check_output(command)
- distutils.log.debug(
+ log.debug(
  "LongPathsAware manifest successfully written into %s", fname
  )
  finally:
  os.remove(manfname)
 
  def build_binary(self, target):
- distutils.log.info("building '%s' binary", target.name)
+ log.info("building '%s' binary", target.name)
  self.build_target(target)
 
  def build_library(self, target):
- distutils.log.info("building '%s' library extension", target.name)
+ log.info("building '%s' library extension", target.name)
  self.build_target(target)
 
  try:
@@ -471,7 +475,7 @@ def update_cargo_env(self, env):
  return env
 
 
-class InstallRustExt(distutils.command.install_scripts.install_scripts):
+class InstallRustExt(install_scripts):
  description = "install Rust extensions and binaries"
 
  def run(self):
@@ -512,7 +516,7 @@ def _callattempt(cmd, env=None, retry=3):
  if retry > 0 and returncode != 0:
  stderr = sio.getvalue()
  if _isflaky(stderr):
- distutils.log.warn("retrying %r on flaky error" % (cmd,))
+ log.warning("retrying %r on flaky error" % (cmd,))
  return _callattempt(cmd, env=env, retry=retry - 1)
  return returncode
 
@@ -528,16 +532,16 @@ def _isflaky(stderr):
  return False
 
 
-distutils.dist.Distribution.rust_ext_modules = ()
-distutils.dist.Distribution.rust_ext_binaries = ()
-distutils.command.build.build.sub_commands.append(
+Distribution.rust_ext_modules = ()
+Distribution.rust_ext_binaries = ()
+build.sub_commands.append(
  (
  "build_rust_ext",
  lambda self: bool(self.distribution.rust_ext_modules)
  or bool(self.distribution.rust_ext_binaries),
  )
 )
-distutils.command.install.install.sub_commands.append(
+install.sub_commands.append(
  (
  "install_rust_ext",
  lambda self: bool(self.distribution.rust_ext_modules)

eden/scm/sapling/util.py

@@ -2254,9 +2254,13 @@ def makedate(timestamp=None):
  if timestamp < 0:
  hint = _("check your clock")
  raise Abort(_("negative timestamp: %d") % timestamp, hint=hint)
- delta = datetime.datetime.utcfromtimestamp(
- timestamp
- ) - datetime.datetime.fromtimestamp(timestamp)
+ as_utc = datetime.datetime.fromtimestamp(
+ timestamp,
+ datetime.timezone.utc
+ )
+ converted = as_utc.astimezone()
+ forced = as_utc.replace(tzinfo=converted.tzinfo)
+ delta = forced - converted
  tz = delta.days * 86400 + delta.seconds
  return timestamp, tz