From f166736bb5a1e81d6abea5199bc7aa10b639efa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kasper=20Ga=C5=82kowski?= Date: Sun, 16 Jun 2024 14:03:52 +0200 Subject: [PATCH 01/22] adding scratch progress on nix profile changes etc --- asdf-nix.patch | 54 ++++++++++++ babel-tmp.patch | 19 ++++ builder.lisp | 73 ++++++++++++++- cl-readline-tmp.patch | 10 +++ default.nix | 3 + nix-cl.nix | 66 +++++++++++--- packages.nix | 186 +++++++-------------------------------- ql.nix | 11 ++- setup-hook.sh | 91 +++++++++---------- source-location-test.txt | 29 ++++++ 10 files changed, 319 insertions(+), 223 deletions(-) create mode 100644 asdf-nix.patch create mode 100644 babel-tmp.patch create mode 100644 cl-readline-tmp.patch create mode 100644 source-location-test.txt diff --git a/asdf-nix.patch b/asdf-nix.patch new file mode 100644 index 0000000..000f9b4 --- /dev/null +++ b/asdf-nix.patch @@ -0,0 +1,54 @@ +diff --git a/output-translations.lisp b/output-translations.lisp +index d2512ba0..af750f38 100644 +--- a/output-translations.lisp ++++ b/output-translations.lisp +@@ -146,6 +146,7 @@ and the order is by decreasing length of namestring of the source pathname.") + '(environment-output-translations + user-output-translations-pathname + user-output-translations-directory-pathname ++ nix-profile-output-translations + system-output-translations-pathname + system-output-translations-directory-pathname)) + +@@ -184,6 +185,14 @@ and the order is by decreasing length of namestring of the source pathname.") + :direction direction)) + (defun environment-output-translations () + (getenv "ASDF_OUTPUT_TRANSLATIONS")) ++ (defun nix-profile-output-translations () ++ `(:output-translations ++ ,@(loop :for profile :in (reverse (split-string (or (getenv "NIX_PROFILES") ""))) ++ :for share := (subpathname (ensure-directory-pathname profile) "share/") ++ :for conf := (subpathname share *output-translations-directory*) ++ :when (probe-file* conf) ++ :collect `(:include ,conf)) ++ :inherit-configuration)) + + + ;;; Processing the configuration. +diff --git a/source-registry.lisp b/source-registry.lisp +index a086cc88..cc317591 100644 +--- a/source-registry.lisp ++++ b/source-registry.lisp +@@ -187,6 +187,7 @@ after having found a .asd file? True by default.") + user-source-registry + user-source-registry-directory + default-user-source-registry ++ nix-profile-source-registry + system-source-registry + system-source-registry-directory + default-system-source-registry) +@@ -227,6 +228,14 @@ after having found a .asd file? True by default.") + :direction direction)) + (defun environment-source-registry () + (getenv "CL_SOURCE_REGISTRY")) ++ (defun nix-profile-source-registry () ++ `(:source-registry ++ ,@(loop :for profile :in (reverse (split-string (or (getenv "NIX_PROFILES") ""))) ++ :for share := (subpathname (ensure-directory-pathname profile) "share/") ++ :for conf := (subpathname share *source-registry-directory*) ++ :when (probe-file* conf) ++ :collect `(:include ,conf)) ++ :inherit-configuration)) + + + ;;; Process the source-registry configuration diff --git a/babel-tmp.patch b/babel-tmp.patch new file mode 100644 index 0000000..d1db957 --- /dev/null +++ b/babel-tmp.patch @@ -0,0 +1,19 @@ +--- a/babel.asd ++++ a/babel.asd +@@ -52,6 +52,14 @@ + (:file "gbk-map") + (:file "sharp-backslash"))))) + ++(defsystem babel/foo ++ :description "Babel, a charset conversion library." ++ :author "Luis Oliveira " ++ :licence "MIT" ++ :depends-on ("babel") ++ :components ++ ((:file "bla"))) ++ + (defmethod perform ((o test-op) (c (eql (find-system :babel)))) + (operate 'load-op :babel-tests) + (operate 'test-op :babel-tests)) + +Diff finished. Sun Jun 16 11:21:26 2024 diff --git a/builder.lisp b/builder.lisp index ca8fda9..16bd563 100644 --- a/builder.lisp +++ b/builder.lisp @@ -1,14 +1,79 @@ (in-package :cl-user) (load "@asdf@") +(format t "SRC(E): ~A~%" (uiop:getenv "CL_SOURCE_REGISTRY")) +(format t "OUT(E): ~A~%" (uiop:getenv "ASDF_OUTPUT_TRANSLATIONS")) + +;; (when (string= "flexi-streams" (uiop:getenv "pname")) +;; (error "bla")) + +;; TODO warn about uncompiled slashy systems!!! + +;; TODO possibly document how to allow compiling uncompiled slashy systems into +;; "user cache" via an additional runtime output translation configuration. + +(defvar *pwd* (namestring (uiop:getcwd))) + + +(asdf:initialize-source-registry + `(:source-registry :inherit-configuration (:tree ,*pwd*))) +(asdf:initialize-output-translations + `(:output-translations + :disable-cache + ("/nix/store/" "/nix/store/") + (,*pwd* (,*pwd* "__tmpfasl__")) + :inherit-configuration)) + +(defvar *declared-systems* (uiop:split-string (uiop:getenv "systems"))) + +(defvar *system*) + +;; (defmethod asdf/component:around-compile-hook :before ((c asdf:system)) +;; (format t "[INFO] Checking... ~A ~A" c *system*) +;; (unless (member (asdf:primary-system-name c) *declared-systems* :test #'string=) +;; (error "Undeclared dependency on downstream slashy system when loading '~A'. Please override '~A' with system '~A'" *system* (asdf:primary-system-name c) (asdf:component-name c)))) + +;; (defmethod asdf:operate :before (o c &key) +;; (format t "[INFO] Checking... ~A ~A ~A ~A~%" o c *system* (asdf:output-files o c))) + +(defvar *out* (uiop:getenv "out")) + +(defun out-path-p (path) + (or (uiop:string-prefix-p *pwd* (namestring path)) + (uiop:string-prefix-p *out* (namestring path)))) + +(defmethod asdf:perform :before ((o asdf:compile-op) (c asdf:source-file)) + (let ((out (asdf:output-files o c))) + (format t "[INFO] [OUT] Checking... ~A ~A ~A ~A~%" o c *system* out) + ;; (let ((missing (find-if-not #'uiop:probe-file* out))) + (when (find-if-not #'out-path-p out) + (error "While loading '~A': Undeclared subsystem dependency on '~A' in '~A'." *system* (asdf:component-name (asdf:component-system c)) (asdf:primary-system-name c))) + ;; (uiop:ensure-all-directories-exist out) + )) + +;; (defmethod asdf:output-files :around ((o asdf:operation) (c asdf:component)) +;; (loop for file in (call-next-method) +;; for type = (pathname-type file) +;; collect (progn +;; (format t "[CHKFL]: ~A~%" file) +;; (cond +;; ((string= "so" type) +;; #P"BLAAAAAAA") +;; (t file))))) + +(format t "SRC: ~A~%" asdf:*source-registry-parameter*) +(format t "OUT: ~A~%" asdf:*output-translations-parameter*) + (defun load-systems () (handler-case - (let ((systems (uiop:getenv "systems"))) - (dolist (s (uiop:split-string systems :separator " ")) + (dolist (s *declared-systems*) + (let ((*system* s)) (asdf:load-system s))) (error (c) (format t "BUILD FAILED: ~A~%" c) - (uiop:quit 1))) - (uiop:quit 0)) + (error c)))) (load-systems) + +(with-open-file (done ".lisp-build-done" :direction :output) + (write-line "done" done)) diff --git a/cl-readline-tmp.patch b/cl-readline-tmp.patch new file mode 100644 index 0000000..69422ed --- /dev/null +++ b/cl-readline-tmp.patch @@ -0,0 +1,10 @@ +--- a/cl-readline.asd ++++ b/cl-readline.asd +@@ -28,4 +28,5 @@ + (:file "ctypes") + (:file "cl-readline")) + :depends-on (:alexandria ++ :babel/foo + :cffi)) + +Diff finished. Sun Jun 16 11:23:58 2024 diff --git a/default.nix b/default.nix index 5164a61..0c29ed3 100644 --- a/default.nix +++ b/default.nix @@ -16,6 +16,9 @@ let url = "https://gitlab.common-lisp.net/asdf/asdf/-/archive/${version}/asdf-${version}.tar.gz"; hash = "sha256-GCmGUMLniPakjyL/D/aEI93Y6bBxjdR+zxXdSgc9NWo="; }; + patches = [ + ./asdf-nix.patch + ]; installPhase = '' cp build/asdf.lisp $out ''; diff --git a/nix-cl.nix b/nix-cl.nix index 151071f..02ce0d1 100644 --- a/nix-cl.nix +++ b/nix-cl.nix @@ -183,18 +183,18 @@ let asdf = "${asdfFasl}"; }; - preConfigure = '' - source ${./setup-hook.sh} - buildAsdfPath + postPatch = '' + find . -type f -name '*.asd' -and -not -name $pname.asd -delete ''; + LISP = "${pkg}/bin/${program} ${flags}"; + # Build from $src so that go-to-definition works in SLIME/Sly. # Can it be achieved while compiling from $PWD? # See SBCL's :SOURCE-NAMESTRING argument to WITH-COMPILATION-UNIT. buildPhase = optionalString (src != null) '' - export CL_SOURCE_REGISTRY=$CL_SOURCE_REGISTRY:$src// - export ASDF_OUTPUT_TRANSLATIONS="$src:$(pwd):${storeDir}:${storeDir}" - ${pkg}/bin/${program} ${flags} < $buildScript + $LISP < $buildScript + test -f .lisp-build-done ''; # Copy compiled files to store @@ -203,19 +203,54 @@ let # stuff like 'iolib.asdf.asd' for system 'iolib.asd' # # Same with '/': `local-time.asd` for system `cl-postgres+local-time.asd` + + # TODO share source between implementations in nix profile + # around method for prepare-op on source-file? installPhase = let mkSystemsRegex = systems: concatMapStringsSep "\\|" (replaceStrings ["." "+"] ["[.]" "[+]"]) systems; in '' - mkdir -pv $out - cp -r * $out - - # Remove all .asd files except for those in `systems`. - find $out -name "*.asd" \ - | grep -v "/\(${mkSystemsRegex (systems ++ extraAsds)}\)\.asd$" \ - | xargs rm -fv || true + cat < $out/share/common-lisp/asdf-output-translations.conf.d/10-${pname}.conf < $out/share/common-lisp/source-registry.conf.d/10-${pname}.conf < $out/share/common-lisp/systems/${pname}/.cl-source-registry.cache + (:source-registry-cache $asds) + EOF ''; dontPatchShebangs = true; @@ -224,13 +259,16 @@ let # save-lisp-and-die binaries in the past dontStrip = true; - } // (args // { + setupHook = ./setup-hook.sh; + + } // (args // rec { src = if builtins.length (args.patches or []) > 0 then pkgs.applyPatches { inherit (args) src patches; } else args.src; patches = []; propagatedBuildInputs = args.propagatedBuildInputs or [] ++ lispLibs ++ javaLibs ++ nativeLibs; + propagatedUserEnvPkgs = propagatedBuildInputs; }))); # Build the set of lisp packages using `lisp` diff --git a/packages.nix b/packages.nix index aef1d81..52c983f 100644 --- a/packages.nix +++ b/packages.nix @@ -21,27 +21,28 @@ let # source of the second run. # # E.g. cl-unicode creating .txt files during compilation - build-with-compile-into-pwd = args: - let - build = (build-asdf-system (args // { version = args.version + "-build"; })) - .overrideAttrs(o: { - buildPhase = with builtins; '' - mkdir __fasls - export ASDF_OUTPUT_TRANSLATIONS="$(pwd):$(pwd)/__fasls:${storeDir}:${storeDir}" - export CL_SOURCE_REGISTRY=$CL_SOURCE_REGISTRY:$(pwd)// - ${o.pkg}/bin/${o.program} ${o.flags or ""} < ${o.buildScript} - ''; - installPhase = '' - mkdir -pv $out - rm -rf __fasls - cp -r * $out - ''; - }); - in build-asdf-system (args // { - # Patches are already applied in `build` - patches = []; - src = build; - }); + build-with-compile-into-pwd = build-asdf-system # args: + # let + # build = (build-asdf-system (args // { version = args.version + "-build"; })) + # .overrideAttrs(o: { + # buildPhase = with builtins; '' + # mkdir __fasls + # export ASDF_OUTPUT_TRANSLATIONS="$(pwd):$(pwd)/__fasls:${storeDir}:${storeDir}" + # export CL_SOURCE_REGISTRY=$CL_SOURCE_REGISTRY:$(pwd)// + # ${o.pkg}/bin/${o.program} ${o.flags or ""} < ${o.buildScript} + # ''; + # installPhase = '' + # mkdir -pv $out + # rm -rf __fasls + # cp -r * $out + # ''; + # }); + # in build-asdf-system (args // { + # # Patches are already applied in `build` + # patches = []; + # src = build; + # }); + ; # A little hacky isJVM = spec.pkg.pname == "abcl"; @@ -66,7 +67,7 @@ let }; version = "0.24.1"; pname = "cffi"; - lispLibs = with super; [ + lispLibs = with self; [ alexandria babel trivial-features @@ -82,7 +83,7 @@ let }; }; - cl-unicode = build-with-compile-into-pwd { + cl-unicode = build-asdf-system { pname = "cl-unicode"; version = "0.1.6"; src = pkgs.fetchzip { @@ -261,56 +262,6 @@ let lispLibs = super.mathkit.lispLibs ++ [ super.sb-cga ]; }; - nyxt-gtk = build-asdf-system { - inherit (super.nyxt) pname; - version = "2.2.4"; - - lispLibs = super.nyxt.lispLibs ++ (with super; [ - cl-cffi-gtk cl-webkit2 mk-string-metrics cl-css - ]); - - src = pkgs.fetchzip { - url = "https://github.com/atlas-engineer/nyxt/archive/2.2.4.tar.gz"; - sha256 = "12l7ir3q29v06jx0zng5cvlbmap7p709ka3ik6x29lw334qshm9b"; - }; - - buildInputs = [ - pkgs.makeWrapper - - # needed for GSETTINGS_SCHEMAS_PATH - pkgs.gsettings-desktop-schemas pkgs.glib pkgs.gtk3 - - # needed for XDG_ICON_DIRS - pkgs.gnome.adwaita-icon-theme - ]; - - buildScript = pkgs.writeText "build-nyxt.lisp" '' - (load "${spec.asdf}") - (asdf:load-system :nyxt/gtk-application) - (sb-ext:save-lisp-and-die "nyxt" :executable t - #+sb-core-compression :compression - #+sb-core-compression t - :toplevel #'nyxt:entry-point) - ''; - - # Run with WEBKIT_FORCE_SANDBOX=0 if getting a runtime error - # See https://github.com/atlas-engineer/nyxt/issues/1781 - # TODO(kasper): use wrapGAppsHook - installPhase = super.nyxt.installPhase + '' - rm -v $out/nyxt - mkdir -p $out/bin - cp -v nyxt $out/bin - wrapProgram $out/bin/nyxt \ - --prefix LD_LIBRARY_PATH : $LD_LIBRARY_PATH \ - --prefix XDG_DATA_DIRS : $XDG_ICON_DIRS \ - --prefix XDG_DATA_DIRS : $GSETTINGS_SCHEMAS_PATH \ - --prefix GIO_EXTRA_MODULES ":" ${pkgs.dconf.lib}/lib/gio/modules/ \ - --prefix GIO_EXTRA_MODULES ":" ${pkgs.glib-networking}/lib/gio/modules/ - ''; - }; - - nyxt = self.nyxt-gtk; - ltk = super.ltk.overrideLispAttrs (o: { src = pkgs.fetchzip { url = "https://github.com/uthar/ltk/archive/f19162e76d6c7c2f51bd289b811d9ba20dd6555e.tar.gz"; @@ -319,88 +270,15 @@ let version = "f19162e76"; }); - - qt = let - rev = "dffff3ee3dbd0686c85c323f579b8bbf4881e60e"; - in build-with-compile-into-pwd rec { - pname = "commonqt"; - version = builtins.substring 0 7 rev; - src = pkgs.fetchFromGitHub { - inherit rev; - owner = pname; - repo = pname; - hash = "sha256-GAgwT0D9mIkYPTHfCH/KxxIv7b6QGwcxwZE7ehH5xug="; - }; - - buildInputs = [ pkgs.qt4 ]; - nativeBuildInputs = [ pkgs.smokegen pkgs.smokeqt ]; - nativeLibs = [ pkgs.qt4 pkgs.smokegen pkgs.smokeqt ]; - - systems = [ "qt" ]; - - lispLibs = with super; [ - cffi named-readtables cl-ppcre alexandria - closer-mop iterate trivial-garbage bordeaux-threads - ]; - }; - - qt-libs = build-with-compile-into-pwd { - inherit (super.qt-libs) pname version src; - patches = [ ./patches/qt-libs-dont-download.patch ]; - prePatch = '' - substituteInPlace systems/*.asd --replace ":qt+libs" ":qt" - ''; - lispLibs = super.qt-libs.lispLibs ++ [ self.qt ]; - systems = [ - "qt-libs" - "commonqt" - # "phonon" - # "qimageblitz" - # "qsci" - "qt3support" - "qtcore" - "qtdbus" - "qtdeclarative" - "qtgui" - "qthelp" - "qtnetwork" - "qtopengl" - "qtscript" - "qtsql" - "qtsvg" - "qttest" - "qtuitools" - # "qtwebkit" - "qtxml" - "qtxmlpatterns" - # "qwt" - "smokebase" - ]; - }; + babel = super.babel.overrideLispAttrs (o: { + patches = [ ./babel-tmp.patch ]; + postPatch = "echo '(defun foo () 42)' > bla.lisp"; + # systems = o.systems ++ [ "babel/foo" ]; + }); - commonqt = self.qt-libs; - qt3support = self.qt-libs; - qtcore = self.qt-libs; - qtdbus = self.qt-libs; - qtdeclarative = self.qt-libs; - qtgui = self.qt-libs; - qthelp = self.qt-libs; - qtnetwork = self.qt-libs; - qtopengl = self.qt-libs; - qtscript = self.qt-libs; - qtsql = self.qt-libs; - qtsvg = self.qt-libs; - qttest = self.qt-libs; - qtuitools = self.qt-libs; - qtxml = self.qt-libs; - qtxmlpatterns = self.qt-libs; - smokebase = self.qt-libs; - - qtools = build-with-compile-into-pwd { - inherit (super.qtools) pname version src nativeLibs; - lispLibs = [ self.qt ] ++ remove super.qt_plus_libs super.qtools.lispLibs ++ [ self.qt-libs ]; - patches = [ ./patches/qtools-use-nix-libs.patch ]; - }; + cl-readline = super.cl-readline.overrideLispAttrs (o: { + patches = [ ./cl-readline-tmp.patch ]; + }); magicl = build-with-compile-into-pwd { inherit (super.magicl) pname version src lispLibs; diff --git a/ql.nix b/ql.nix index b3ce1a9..a74d104 100644 --- a/ql.nix +++ b/ql.nix @@ -6,7 +6,7 @@ let overrides = (self: super: { cl_plus_ssl = super.cl_plus_ssl.overrideLispAttrs (o: { - nativeLibs = [ pkgs.openssl ]; + nativeLibs = [ pkgs.openssl.out ]; }); cl-cffi-gtk-glib = super.cl-cffi-gtk-glib.overrideLispAttrs (o: { nativeLibs = [ pkgs.glib ]; @@ -148,12 +148,12 @@ let nativeLibs = [ pkgs.rdkafka ]; }); cl-async-ssl = super.cl-async-ssl.overrideLispAttrs (o: { - nativeLibs = [ pkgs.openssl ]; + nativeLibs = [ pkgs.openssl.out ]; }); iolib = super.iolib.overrideLispAttrs (o: { nativeBuildInputs = [ pkgs.libfixposix ]; nativeLibs = [ pkgs.libfixposix ]; - systems = [ "iolib" "iolib/os" "iolib/pathnames" ]; + systems = [ "iolib" ]; }); cl-ana_dot_hdf-cffi = super.cl-ana_dot_hdf-cffi.overrideLispAttrs (o: { nativeBuildInputs = [ pkgs.hdf5 ]; @@ -170,7 +170,7 @@ let cl-libxml2 = super.cl-libxml2.overrideLispAttrs (o: { nativeLibs = [ pkgs.libxml2 ]; }); - cl-readline = super.cl-readline.overrideLispAttrs (o: { + cl-readline = super.lc-readline.overrideLispAttrs (o: { nativeLibs = [ pkgs.readline ]; }); md5 = super.md5.overrideLispAttrs (o: { @@ -205,6 +205,9 @@ let cl-sat_dot_minisat = super.cl-sat_dot_minisat.overrideLispAttrs (o: { propagatedBuildInputs = [ pkgs.minisat ]; }); + mssql = super.mssql.overrideLispAttrs (o: { + nativeLibs = [ pkgs.freetds ]; + }); }); qlpkgs = diff --git a/setup-hook.sh b/setup-hook.sh index 6beb962..0f162a4 100644 --- a/setup-hook.sh +++ b/setup-hook.sh @@ -1,56 +1,53 @@ -# This setup hook adds every propagated lisp system to CL_SOURCE_REGISTRY -buildAsdfPath () { - declare -A seen=() - for dep in $propagatedBuildInputs; do - _addToAsdfPath $dep - done -} - -addFileToSearchPathWithCustomDelimiter() { - local delimiter="$1" - local varName="$2" - local file="$3" - if [[ -f "$file" && "${!varName:+${delimiter}${!varName}${delimiter}}" \ - != *"${delimiter}${file}${delimiter}"* ]]; then - export "${varName}=${!varName:+${!varName}${delimiter}}${file}" +# TODO: it might be faster to create files in temp conf dir in /build, then set +# CL_SOURCE_REGISTRY to just (:import "/build/source-registry.conf.d") +# ASDF_OUTPUT_TRANSLATIONS to just (:import "/build/output-translations.conf.d") +# +# How about nix-shell/dev shell? Is it fine to create files there, somewhere like /tmp? + +addAsdfSourceRegistry () { + if test -z "${CL_SOURCE_REGISTRY:-}"; then + export CL_SOURCE_REGISTRY="(:source-registry :ignore-inherited-configuration)" + fi + if test -d "$1/share/common-lisp/source-registry.conf.d/"; then + if [[ ! "$CL_SOURCE_REGISTRY" =~ "$1/share/common-lisp/source-registry.conf.d/" ]]; then + export CL_SOURCE_REGISTRY="(:source-registry :ignore-inherited-configuration (:include \"$1/share/common-lisp/source-registry.conf.d/\")${CL_SOURCE_REGISTRY:49}" fi + fi } -addFileToSearchPath() { - addFileToSearchPathWithCustomDelimiter ":" "$@" +addAsdfOutputTranslation () { + if test -z "${ASDF_OUTPUT_TRANSLATIONS:-}"; then + export ASDF_OUTPUT_TRANSLATIONS="(:output-translations :ignore-inherited-configuration)" + fi + if test -d "$1/share/common-lisp/asdf-output-translations.conf.d/"; then + if [[ ! "$ASDF_OUTPUT_TRANSLATIONS" =~ "$1/share/common-lisp/asdf-output-translations.conf.d/" ]]; then + export ASDF_OUTPUT_TRANSLATIONS="(:output-translations :ignore-inherited-configuration (:include \"$1/share/common-lisp/asdf-output-translations.conf.d/\")${ASDF_OUTPUT_TRANSLATIONS:53}" + fi + fi } -_addToAsdfPath () { - local dep="$1" - if [ -v seen[$dep] ]; then - return - else - seen[$dep]=1 - local path="$dep" - - # FIXME slow - - while read file; do - case "${file##*.}" in - jar) addFileToSearchPath "CLASSPATH" "$file" ;; - class) addToSearchPath "CLASSPATH" "${file%/*}" ;; - so) addToSearchPath "LD_LIBRARY_PATH" "${file%/*}" ;; - dylib) addToSearchPath "DYLD_LIBRARY_PATH" "${file%/*}" ;; - asd) addToSearchPath "CL_SOURCE_REGISTRY" "$path//" ;; - esac - done < <(find "$path" -type f,l -name '*.asd' -o -name '*.jar' \ - -o -name '*.class' -o -name '*.so' -o -name '*.dylib') - - local prop="$dep/nix-support/propagated-build-inputs" - - if [ -e "$prop" ]; then - local new_system - for new_system in $(cat $prop); do - _addToAsdfPath "$new_system" - done - fi +addLibToLibraryPath () { + if test -z "${LD_LIBRARY_PATH:-}"; then + export LD_LIBRARY_PATH="" + fi + if test -z "${DYLD_LIBRARY_PATH:-}"; then + export DYLD_LIBRARY_PATH="" + fi + if test -d "$1/lib"; then + if test -n "$(find "$1/lib" -name '*.so' -print -quit)"; then + if [[ ! "$LD_LIBRARY_PATH" =~ "$1/lib" ]]; then + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$1/lib" + fi + fi + if test -n "$(find "$1/lib" -name '*.dylib' -print -quit)"; then + if [[ ! "$DYLD_LIBRARY_PATH" =~ "$1/lib" ]]; then + export DYLD_LIBRARY_PATH="${DYLD_LIBRARY_PATH:+$DYLD_LIBRARY_PATH:}$1/lib" + fi fi + fi } -# addEnvHooks "$targetOffset" buildAsdfPath +addEnvHooks "$hostOffset" addAsdfSourceRegistry +addEnvHooks "$hostOffset" addAsdfOutputTranslation +addEnvHooks "$hostOffset" addLibToLibraryPath diff --git a/source-location-test.txt b/source-location-test.txt new file mode 100644 index 0000000..d1fedbb --- /dev/null +++ b/source-location-test.txt @@ -0,0 +1,29 @@ +/nix/store/wkq6jbjb7vxqcfkz2smj92ayplhq90n7-abcl-abcl-with-packages +(load (ext:getenv "ASDF")) +(asdf:load-system 'clop) +(get 'clop:parse 'sys::source) + +/nix/store/zb6ba7j678ysqwqbncp6a9blxiq35bmh-ccl-ccl-with-packages +(load (ccl:getenv "ASDF")) +(asdf:load-system 'clop) +(ext:source-location 'clop:parse :function) + +/nix/store/v7r5wfz81vx0plb5qfpqbd50i7n41w9h-clisp-clisp-with-packages +(load (ext:getenv "ASDF")) +(asdf:load-system 'com.inuoe.jzon) +(documentation 'com.inuoe.jzon:parse 'sys::file) + +/nix/store/w0bms0yfwr84sr8dwa3yj260giky1amn-clasp-clasp-with-packages +(load (ext:getenv "ASDF")) +(asdf:load-system 'clop) +(ext:source-location 'clop:parse :function) + +/nix/store/9h1zhwk2i9ii7yf8wjnzpym1k0nwbgi5-ecl-ecl-with-packages +(load (ext:getenv "ASDF")) +(asdf:load-system 'clop) +(ext:get-annotation 'clop:parse 'si::location :all) + +/nix/store/lby5g1hqjw993xpbh79q3kibkf0qdmwa-sbcl-sbcl-with-packages +(load (sb-ext:posix-getenv "ASDF")) +(asdf:load-system 'clop) +(sb-introspect:find-definition-sources-by-name 'clop:parse :function) From b210af7a6f669999202db9714f1bbf93ca0e7350 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kasper=20Ga=C5=82kowski?= Date: Mon, 17 Jun 2024 23:37:35 +0200 Subject: [PATCH 02/22] add early scratch of metadata scraping --- import/init.sql | 9 ++ import/org.lispbuilds.nix.asd | 2 + import/repository/quicklisp.lisp | 231 ++++++++++++++++++++++++++----- import/util.lisp | 18 ++- setup-hook.sh | 16 +-- shell.nix | 2 +- 6 files changed, 230 insertions(+), 48 deletions(-) diff --git a/import/init.sql b/import/init.sql index 872d51d..148cb2e 100644 --- a/import/init.sql +++ b/import/init.sql @@ -2,6 +2,7 @@ CREATE TABLE IF NOT EXISTS sha256 ( id integer PRIMARY KEY AUTOINCREMENT, url text UNIQUE, hash text NOT NULL, + path text NOT NULL, created real DEFAULT (julianday('now')) ); @@ -25,6 +26,14 @@ CREATE TABLE IF NOT EXISTS src ( system_id integer UNIQUE REFERENCES system(id) ); +CREATE TABLE IF NOT EXISTS meta ( + system_id integer UNIQUE REFERENCES system(id), + homepage text, + license text, + description text, + long_description text +); + DROP VIEW IF EXISTS system_view; CREATE VIEW IF NOT EXISTS system_view AS SELECT diff --git a/import/org.lispbuilds.nix.asd b/import/org.lispbuilds.nix.asd index ebf3851..e154721 100644 --- a/import/org.lispbuilds.nix.asd +++ b/import/org.lispbuilds.nix.asd @@ -6,9 +6,11 @@ :str :cl-ppcre :sqlite + :tar :dexador :arrow-macros :com.inuoe.jzon + :sb-concurrency :org.lispbuilds.nix/api :org.lispbuilds.nix/repository/quicklisp :org.lispbuilds.nix/database/sqlite diff --git a/import/repository/quicklisp.lisp b/import/repository/quicklisp.lisp index 3a45e06..28d436b 100644 --- a/import/repository/quicklisp.lisp +++ b/import/repository/quicklisp.lisp @@ -1,9 +1,10 @@ (defpackage org.lispbuilds.nix/repository/quicklisp (:use :cl) (:import-from :dex) - (:import-from :alexandria :read-file-into-string :ensure-list) + (:import-from :alexandria :read-file-into-string :ensure-list :when-let :ensure-gethash) (:import-from :arrow-macros :->>) (:import-from :str) + (:import-from :tar) (:import-from :org.lispbuilds.nix/database/sqlite :sqlite-database @@ -15,6 +16,9 @@ :import-lisp-packages) (:import-from :org.lispbuilds.nix/util + :make-count-down-latch + :count-down + :await :replace-regexes) (:export :quicklisp-repository) (:local-nicknames @@ -38,6 +42,9 @@ (apply #'format (list* *error-output* format-args)) (force-output *error-output*)) +(defvar *db* nil + "Current SQLite connection") + ;; TODO: This should not know about the imported.nix file. (defun init-tarball-hashes (database) (status "no packages.sqlite - will pre-fill tarball hashes from ~A to save time~%" @@ -56,18 +63,21 @@ (declare (ignore whole)) (svref groups 0))) lines))) - (sqlite:with-open-database (db (database-url database)) - (init-db db (init-file database)) - (sqlite:with-transaction db + (sqlite:with-open-database (*db* (database-url database)) + (init-db *db* (init-file database)) + (sqlite:with-transaction *db* (loop while lines do - (sqlite:execute-non-query db + (sqlite:execute-non-query *db* "insert or ignore into sha256(url,hash) values (?,?)" (prog1 (first lines) (setf lines (rest lines))) (prog1 (first lines) (setf lines (rest lines)))))) (status "OK, imported ~A hashes into DB.~%" - (sqlite:execute-single db + (sqlite:execute-single *db* "select count(*) from sha256"))))) +(defun sql-query (sql &rest params) + (apply #'sqlite:execute-to-list (list* *db* sql params))) + (defmethod import-lisp-packages ((repository quicklisp-repository) (database sqlite-database)) @@ -76,17 +86,16 @@ (unless (probe-file (database-url database)) (init-tarball-hashes database)) - (let* ((db (sqlite:connect (database-url database))) + (let* ((*db* (sqlite:connect (database-url database))) (systems-url (str:concat (dist-url repository) "systems.txt")) (releases-url (str:concat (dist-url repository) "releases.txt")) (systems-lines (rest (butlast (str:split #\Newline (dex:get systems-url))))) (releases-lines (rest (butlast (str:split #\Newline (dex:get releases-url)))))) - (flet ((sql-query (sql &rest params) - (apply #'sqlite:execute-to-list (list* db sql params)))) + (progn ;; Ensure database schema - (init-db db (init-file database)) + (init-db *db* (init-file database)) ;; Prepare temporary tables for efficient access (sql-query "create temp table if not exists quicklisp_system @@ -95,7 +104,7 @@ (sql-query "create temp table if not exists quicklisp_release (project unique, url, size, md5, sha1, prefix not null, asds)") - (sqlite:with-transaction db + (sqlite:with-transaction *db* (dolist (line systems-lines) (destructuring-bind (project asd name &rest deps) (str:words line) @@ -103,7 +112,7 @@ "insert or ignore into quicklisp_system values(?,?,?,?)" project asd name (json:stringify (coerce deps 'vector)))))) - (sqlite:with-transaction db + (sqlite:with-transaction *db* (dolist (line releases-lines) (destructuring-bind (project url size md5 sha1 prefix &rest asds) (str:words line) @@ -113,7 +122,7 @@ asds 'vector)))))) - (sqlite:with-transaction db + (progn ;; Should these be temp tables, that then get queried by ;; system name? This looks like it uses a lot of memory. (let ((systems @@ -131,7 +140,7 @@ json_array(value, (select version from pkg where name=value)) ) from json_each(deps)) as deps - from pkg" + from pkg where name not in (select name from system)" ))) ;; First pass: insert system and source tarball informaton. @@ -139,22 +148,25 @@ ;; on system ids in the database and they don't exist ;; yet. Could it be better to just base dependencies on ;; names? But then ACID is lost. - (dolist (system systems) - (destructuring-bind (name version asd url deps) system - (declare (ignore deps)) - (status "importing system '~a-~a'" name version) - (let ((hash (nix-prefetch-tarball url db))) - (sql-query - "insert or ignore into system(name,version,asd) values (?,?,?)" - name version asd) - (sql-query - "insert or ignore into sha256(url,hash) values (?,?)" - url hash) - (sql-query - "insert or ignore into src values - ((select id from sha256 where url=?), - (select id from system where name=? and version=?))" - url name version)))) + (let ((tgz-jobs (sb-concurrency:make-mailbox)) + (tgz-done (sb-concurrency:make-mailbox)) + (tgz-failed (sb-concurrency:make-mailbox)) + (asd-jobs (sb-concurrency:make-mailbox)) + (asd-done (sb-concurrency:make-mailbox)) + (asd-failed (sb-concurrency:make-mailbox)) + (*bus* (sb-concurrency:make-mailbox)) + (scrape-done (make-count-down-latch :count (length systems)))) + (tgz-scheduler) + (asd-scheduler) + (retry-sheduler) + (tgz-scraper) + (asd-scraper) + (tgz-loader) + (asd-loader) + (dolist (system systems) + (sb-concurrency:send-message tarball-jobs system)) + (sb-concurrency:send-message tarball-jobs nil) + (await scrape-done)) ;; Second pass: connect the in-database systems with ;; dependency information @@ -171,9 +183,94 @@ (select id from system where name=? and version=?))" name version dep-name dep-version)))))))))) - (write-char #\Newline *error-output*)) +(defun download-tarballs (jobs scrape-jobs) + (bt:make-thread + (lambda () + (unwind-protect + (let ((limit (bt:make-semaphore :count 2))) + (loop for job = (sb-concurrency:receive-message jobs) while job do + (bt:wait-on-semaphore limit) + (bt:make-thread + (destructuring-bind (name version asd url deps) job + (declare (ignore deps)) + (lambda () + (status "importing system BLA '~a-~a'" name version) + (let (tgz-path) + (unwind-protect + (let ((res (fetch-tarball url))) + (destructuring-bind (hash path) res + (setf tgz-path (truename path)) + (sql-query + "insert or ignore into sha256(url,hash,path) values (?,?,?)" + url hash path))) + (bt:signal-semaphore limit)) + (unwind-protect + (sql-query + "insert or ignore into system(name,version,asd) values (?,?,?)" + name version asd) + (sb-concurrency:send-message scrape-jobs (cons tgz-path job)))) + (sql-query + "insert or ignore into src values + ((select id from sha256 where url=?), + (select id from system where name=? and version=?))" + url name version)))))) + (format t "tgz downloading done.~%") + (sb-concurrency:send-message scrape-jobs nil))))) + +(defun scrape-asds (jobs done) + (bt:make-thread + (lambda () + (unwind-protect + (let ((retries (make-hash-table :test 'equal :synchronized t))) + (loop for job = (sb-concurrency:receive-message jobs) while job do + (destructuring-bind (path name version asd url deps) job + (declare (ignore url deps)) + (let (mark-done) + (unwind-protect + (handler-case + (let (sys found-asd short long license homepage) + (setf found-asd (find-file asd path)) + (asdf:load-asd found-asd) + (setf sys (asdf:find-system name t)) + (setf short (asdf:system-description sys)) + ;; It's too long - whole README sometimes + ;; (setf long (asdf:system-long-description sys)) + (when-let ((lic (or (asdf:system-licence sys) + (asdf:system-license sys)))) + (setf license (string lic))) ;sometimes a keyword + (when-let ((home (asdf:system-homepage sys))) + (when (stringp home) ;sometimes a symbol + (setf homepage home))) + (sql-query + "insert or replace into meta values + ((select id from system where name=? and version=?) + ,?,?,?,?)" + name version homepage license short long) + (setf mark-done t)) + (file-not-found (e) + (format t "E: ~A~%" e) + (setf mark-done t)) + (asdf:missing-dependency (e) + (format t "E: ~A~%" e) + (ensure-gethash name retries 0) + (if (<= (incf (gethash name retries)) 10) + (sb-concurrency:send-message jobs job) + (setf mark-done t))) + (asdf:missing-component (e) + (format t "E: ~A~%" e) + (setf mark-done t)) + (error (e) + (format t "E: ~A~%" e) + (ensure-gethash name retries 0) + (if (<= (incf (gethash name retries)) 10) + (sb-concurrency:send-message jobs job) + (setf mark-done t)))) + (when mark-done + (count-down done))))))) + (format t "asd scraping done.~%"))))) + (defun shell-command-to-string (cmd) ;; Clearing the library path is needed to prevent a bug, where the ;; called subprocess uses a different glibc than the SBCL process @@ -186,14 +283,72 @@ (uiop:run-program cmd :output '(:string :stripped t)) (setf (uiop:getenv "LD_LIBRARY_PATH") ld-library-path)))) -(defun nix-prefetch-tarball (url db) +(defun fetch-tarball (url) (restart-case - (compute-sha256 url db) + (prefetch-tarball-cached url) (try-again () :report "Try downloading again" - (nix-prefetch-tarball url db)))) + (fetch-tarball url)))) + +(defvar *up-for-retry* + (make-hash-table :test 'equal :synchronized t)) + +(defun find-file (what where) + (or + (dolist (file (uiop:directory-files where)) + (when (and (string= (pathname-name file) (pathname-name what)) + (string= (pathname-type file) (pathname-type what))) + (return file))) + (dolist (dir (uiop:subdirectories where)) + (when-let ((found (find-file what dir))) + (return found))))) + +(define-condition file-not-found (error) + ((file :initarg :file :reader .file))) + +(defun analyze-tarball (url system asd) + (multiple-value-bind (hash path) + (nix-prefetch-tarball url) + (restart-case + (let (sys res found-asd) + (setf found-asd (find-file asd path)) + (when (null found-asd) + (error 'file-not-found :file asd)) + (asdf:load-asd found-asd) + (setf sys (asdf:find-system system t)) + (setf (getf res :hash) hash) + (when-let ((short (asdf:system-description sys))) + (setf (getf res :short) short)) + (when-let ((long (asdf:system-long-description sys))) + (setf (getf res :long) long)) + (when-let ((lic (or (asdf:system-licence sys) + (asdf:system-license sys)))) + ;; sometimes a keyword + (setf (getf res :license) (string lic))) + (when-let ((home (asdf:system-homepage sys))) + (setf (getf res :home) home)) + res) + (retry () + (format t "Retrying to get metadata about: ~A~%" system) + (analyze-tarball url system asd)) + (retry-later (c) + (format t "Will try to scrape ~A after its defsystem dependencies are loaded~%" asd) + (vector-push-extend + (list :url url :system system :asd asd) + (ensure-gethash (asdf/find-component:missing-requires c) + *up-for-retry* + (make-array 4 :adjustable t :fill-pointer 0))) + (list :hash hash)) + (skip () + (format t "Will not scrape metadata about: ~A~%" system) + (list :hash hash))))) + +(declaim (optimize (debug 3))) -(defun compute-sha256 (url db) - (or (sqlite:execute-single db "select hash from sha256 where url=?" url) - (let ((sha256 (shell-command-to-string (str:concat "nix-prefetch-url --unpack " url)))) - sha256))) +(defun prefetch-tarball-cached (url) + (or (first (sqlite:execute-to-list *db* "select hash, path from sha256 where url = ?;" url)) + (let* ((cmd (str:concat "nix-prefetch-url --print-path --unpack " url)) + (res (shell-command-to-string cmd))) + (destructuring-bind (hash path) + (uiop:split-string res :separator '(#\Newline)) + (list hash path))))) diff --git a/import/util.lisp b/import/util.lisp index 0432763..0257125 100644 --- a/import/util.lisp +++ b/import/util.lisp @@ -1,8 +1,12 @@ (defpackage org.lispbuilds.nix/util (:use :cl) (:import-from :ppcre) + (:import-from :sb-concurrency) (:export - :replace-regexes)) + :replace-regexes + :make-count-down-latch + :count-down + :await)) (in-package org.lispbuilds.nix/util) @@ -14,3 +18,15 @@ (rest from) (rest to) (ppcre:regex-replace-all (first from) str (first to))))) + +(defstruct (count-down-latch (:conc-name %latch-)) + (gate (sb-concurrency:make-gate) :type sb-concurrency:gate) + (count (error "count required") :type (unsigned-byte 64))) + +(defun count-down (latch &optional (amount 1)) + (when (<= (sb-ext:atomic-decf (%latch-count latch) amount) amount) + (sb-concurrency:open-gate (%latch-gate latch)))) + +(defun await (latch &optional timeout) + (sb-concurrency:wait-on-gate (%latch-gate latch) :timeout timeout)) + diff --git a/setup-hook.sh b/setup-hook.sh index 0f162a4..a6df324 100644 --- a/setup-hook.sh +++ b/setup-hook.sh @@ -6,14 +6,14 @@ # How about nix-shell/dev shell? Is it fine to create files there, somewhere like /tmp? addAsdfSourceRegistry () { - if test -z "${CL_SOURCE_REGISTRY:-}"; then - export CL_SOURCE_REGISTRY="(:source-registry :ignore-inherited-configuration)" - fi - if test -d "$1/share/common-lisp/source-registry.conf.d/"; then - if [[ ! "$CL_SOURCE_REGISTRY" =~ "$1/share/common-lisp/source-registry.conf.d/" ]]; then - export CL_SOURCE_REGISTRY="(:source-registry :ignore-inherited-configuration (:include \"$1/share/common-lisp/source-registry.conf.d/\")${CL_SOURCE_REGISTRY:49}" - fi - fi + export CL_SOURCE_REGISTRY=$(sbcl --script < Date: Sat, 13 Jul 2024 08:38:04 +0200 Subject: [PATCH 03/22] poprawiony setup hook --- setup-hook.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/setup-hook.sh b/setup-hook.sh index a6df324..0f162a4 100644 --- a/setup-hook.sh +++ b/setup-hook.sh @@ -6,14 +6,14 @@ # How about nix-shell/dev shell? Is it fine to create files there, somewhere like /tmp? addAsdfSourceRegistry () { - export CL_SOURCE_REGISTRY=$(sbcl --script < Date: Sat, 13 Jul 2024 17:19:59 +0200 Subject: [PATCH 04/22] Uproszczone ustawienia asdf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wcześniej podpinałem pliki conf z użyciem klauzuli :include. Niestety w ten sposób wymagana jest wyczyszczenie konfiguracji asdf żeby podłapać system nowo dodane do profilu. (Jak się potem okazało, nawet z dodaniem całego katalogu i tak trzeba resetować konfigurację) Zamiast tego mogę przetłumaczyć na sztywno cały katalog systems na fasl. Wymaga to zachowania konwencji gdzie są fasle. Niestety w tym przypadku wymagana jest pełna kopia a nie symlink, chyba że ustawi się asdf:*resolve-symlinks*. Ustawienie samych linków do asdków wymagałoby użycia truename na symlinku, jednak nie wiem jak by to działało gdyby pliki źródłowe znajdowały się w katalogu wyżej niż sam asd. Teraz problemem jest to, że zainstalowanie tej samej biblioteki do profilu kilka razy pod rząd postępująco nadpisuje wcześniejsze pliki - to znaczy że nie są wykrywane konflikty. Może ostatecznie będzie trzeba kopiować źródła. --- asdf-nix.patch | 25 ++++++++++--------------- builder.lisp | 15 ++++++++++++--- nix-cl.nix | 50 ++++++++++++++++---------------------------------- setup-hook.sh | 19 +++++-------------- 4 files changed, 43 insertions(+), 66 deletions(-) diff --git a/asdf-nix.patch b/asdf-nix.patch index 000f9b4..8e8d2c6 100644 --- a/asdf-nix.patch +++ b/asdf-nix.patch @@ -1,8 +1,6 @@ -diff --git a/output-translations.lisp b/output-translations.lisp -index d2512ba0..af750f38 100644 --- a/output-translations.lisp +++ b/output-translations.lisp -@@ -146,6 +146,7 @@ and the order is by decreasing length of namestring of the source pathname.") +@@ -146,6 +146,7 @@ '(environment-output-translations user-output-translations-pathname user-output-translations-directory-pathname @@ -10,7 +8,7 @@ index d2512ba0..af750f38 100644 system-output-translations-pathname system-output-translations-directory-pathname)) -@@ -184,6 +185,14 @@ and the order is by decreasing length of namestring of the source pathname.") +@@ -184,6 +185,14 @@ :direction direction)) (defun environment-output-translations () (getenv "ASDF_OUTPUT_TRANSLATIONS")) @@ -18,18 +16,17 @@ index d2512ba0..af750f38 100644 + `(:output-translations + ,@(loop :for profile :in (reverse (split-string (or (getenv "NIX_PROFILES") ""))) + :for share := (subpathname (ensure-directory-pathname profile) "share/") -+ :for conf := (subpathname share *output-translations-directory*) -+ :when (probe-file* conf) -+ :collect `(:include ,conf)) ++ :for sysdir := (namestring (subpathname share "common-lisp/systems/")) ++ :for fasldir := (namestring (subpathname share "common-lisp/fasl/")) ++ :collect `(,sysdir (,fasldir :implementation))) + :inherit-configuration)) ;;; Processing the configuration. -diff --git a/source-registry.lisp b/source-registry.lisp -index a086cc88..cc317591 100644 + --- a/source-registry.lisp +++ b/source-registry.lisp -@@ -187,6 +187,7 @@ after having found a .asd file? True by default.") +@@ -187,6 +187,7 @@ user-source-registry user-source-registry-directory default-user-source-registry @@ -37,7 +34,7 @@ index a086cc88..cc317591 100644 system-source-registry system-source-registry-directory default-system-source-registry) -@@ -227,6 +228,14 @@ after having found a .asd file? True by default.") +@@ -227,5 +228,11 @@ :direction direction)) (defun environment-source-registry () (getenv "CL_SOURCE_REGISTRY")) @@ -45,10 +42,8 @@ index a086cc88..cc317591 100644 + `(:source-registry + ,@(loop :for profile :in (reverse (split-string (or (getenv "NIX_PROFILES") ""))) + :for share := (subpathname (ensure-directory-pathname profile) "share/") -+ :for conf := (subpathname share *source-registry-directory*) -+ :when (probe-file* conf) -+ :collect `(:include ,conf)) ++ :collect `(:tree ,(namestring (subpathname share (parse-unix-namestring "common-lisp/systems/"))))) + :inherit-configuration)) - ;;; Process the source-registry configuration + ;;; Process the source-registry configuration \ No newline at end of file diff --git a/builder.lisp b/builder.lisp index 16bd563..e129ed7 100644 --- a/builder.lisp +++ b/builder.lisp @@ -13,15 +13,20 @@ ;; "user cache" via an additional runtime output translation configuration. (defvar *pwd* (namestring (uiop:getcwd))) +(defvar *src* (namestring (uiop:getenv "src"))) +(defvar *out* (namestring (uiop:getenv "out"))) +(defvar *pname* (format nil "~a/" (uiop:getenv "pname"))) +(setf asdf:*resolve-symlinks* nil) (asdf:initialize-source-registry - `(:source-registry :inherit-configuration (:tree ,*pwd*))) + `(:source-registry :inherit-configuration (:tree ,*src*))) + (asdf:initialize-output-translations `(:output-translations :disable-cache ("/nix/store/" "/nix/store/") - (,*pwd* (,*pwd* "__tmpfasl__")) + (,*src* (,*out* "share" "common-lisp" "fasl" :implementation ,*pname*)) :inherit-configuration)) (defvar *declared-systems* (uiop:split-string (uiop:getenv "systems"))) @@ -36,12 +41,16 @@ ;; (defmethod asdf:operate :before (o c &key) ;; (format t "[INFO] Checking... ~A ~A ~A ~A~%" o c *system* (asdf:output-files o c))) -(defvar *out* (uiop:getenv "out")) (defun out-path-p (path) (or (uiop:string-prefix-p *pwd* (namestring path)) (uiop:string-prefix-p *out* (namestring path)))) +(defmethod asdf:perform :before ((o asdf:prepare-op) (c asdf:file-component)) + (let ((path (asdf:component-relative-pathname c)) + (path2 (asdf:component-pathname c))) + (format t "FOOOOOOO: ~A >>> ~A~%" path path2))) + (defmethod asdf:perform :before ((o asdf:compile-op) (c asdf:source-file)) (let ((out (asdf:output-files o c))) (format t "[INFO] [OUT] Checking... ~A ~A ~A ~A~%" o c *system* out) diff --git a/nix-cl.nix b/nix-cl.nix index 02ce0d1..c7dc5ce 100644 --- a/nix-cl.nix +++ b/nix-cl.nix @@ -212,28 +212,6 @@ let concatMapStringsSep "\\|" (replaceStrings ["." "+"] ["[.]" "[+]"]) systems; in '' - cat < $out/share/common-lisp/asdf-output-translations.conf.d/10-${pname}.conf < $out/share/common-lisp/source-registry.conf.d/10-${pname}.conf < $out/share/common-lisp/systems/${pname}/.cl-source-registry.cache - (:source-registry-cache $asds) - EOF ''; dontPatchShebangs = true; @@ -262,10 +234,20 @@ let setupHook = ./setup-hook.sh; } // (args // rec { - src = if builtins.length (args.patches or []) > 0 - then pkgs.applyPatches { inherit (args) src patches; } - else args.src; + src = pkgs.applyPatches { + inherit (args) src; + systems = args.systems or [ args.pname ]; + patches = args.patches or []; + postPatch = '' + declare -a asds=() + for s in $systems; do + asds+="\"$(find -name $s.asd -type f -print -quit)\"" + done + echo "(:source-registry-cache ''${asds[@]})" > .cl-source-registry.cache + '' + args.postPatch or ""; + }; patches = []; + postPatch = ""; propagatedBuildInputs = args.propagatedBuildInputs or [] ++ lispLibs ++ javaLibs ++ nativeLibs; propagatedUserEnvPkgs = propagatedBuildInputs; @@ -324,7 +306,7 @@ let --add-flags "${o.flags}" \ --set ASDF "${o.asdfFasl}" \ --prefix CL_SOURCE_REGISTRY : "$CL_SOURCE_REGISTRY" \ - --prefix ASDF_OUTPUT_TRANSLATIONS : "$(echo $CL_SOURCE_REGISTRY | sed s,//:,::,g):" \ + --prefix ASDF_OUTPUT_TRANSLATIONS : "$ASDF_OUTPUT_TRANSLATIONS" \ --prefix LD_LIBRARY_PATH : "$LD_LIBRARY_PATH" \ --prefix DYLD_LIBRARY_PATH : "$DYLD_LIBRARY_PATH" \ --prefix CLASSPATH : "$CLASSPATH" \ diff --git a/setup-hook.sh b/setup-hook.sh index 0f162a4..15c2b2a 100644 --- a/setup-hook.sh +++ b/setup-hook.sh @@ -6,24 +6,15 @@ # How about nix-shell/dev shell? Is it fine to create files there, somewhere like /tmp? addAsdfSourceRegistry () { - if test -z "${CL_SOURCE_REGISTRY:-}"; then - export CL_SOURCE_REGISTRY="(:source-registry :ignore-inherited-configuration)" - fi - if test -d "$1/share/common-lisp/source-registry.conf.d/"; then - if [[ ! "$CL_SOURCE_REGISTRY" =~ "$1/share/common-lisp/source-registry.conf.d/" ]]; then - export CL_SOURCE_REGISTRY="(:source-registry :ignore-inherited-configuration (:include \"$1/share/common-lisp/source-registry.conf.d/\")${CL_SOURCE_REGISTRY:49}" - fi + if test -d "$1/share/common-lisp"; then + addToSearchPath CL_SOURCE_REGISTRY "$1/share/common-lisp/systems//" fi } addAsdfOutputTranslation () { - if test -z "${ASDF_OUTPUT_TRANSLATIONS:-}"; then - export ASDF_OUTPUT_TRANSLATIONS="(:output-translations :ignore-inherited-configuration)" - fi - if test -d "$1/share/common-lisp/asdf-output-translations.conf.d/"; then - if [[ ! "$ASDF_OUTPUT_TRANSLATIONS" =~ "$1/share/common-lisp/asdf-output-translations.conf.d/" ]]; then - export ASDF_OUTPUT_TRANSLATIONS="(:output-translations :ignore-inherited-configuration (:include \"$1/share/common-lisp/asdf-output-translations.conf.d/\")${ASDF_OUTPUT_TRANSLATIONS:53}" - fi + if test -d "$1/share/common-lisp"; then + addToSearchPath ASDF_OUTPUT_TRANSLATIONS "$1/share/common-lisp/systems/" + addToSearchPath ASDF_OUTPUT_TRANSLATIONS "$(find $1/share/common-lisp/fasl/ -mindepth 1 -maxdepth 1 -print -quit)/" fi } From aa7a991fa8345582ead311f06a0d52f7106498a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kasper=20Ga=C5=82kowski?= Date: Sun, 14 Jul 2024 23:06:42 +0200 Subject: [PATCH 05/22] =?UTF-8?q?teraz=20cl=20source=20registry=20cache=20?= =?UTF-8?q?powinien=20dzia=C5=82a=C4=87=20z=20wieloma=20systemami?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nix-cl.nix | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nix-cl.nix b/nix-cl.nix index c7dc5ce..79cf862 100644 --- a/nix-cl.nix +++ b/nix-cl.nix @@ -239,11 +239,12 @@ let systems = args.systems or [ args.pname ]; patches = args.patches or []; postPatch = '' - declare -a asds=() + declare -A asds for s in $systems; do - asds+="\"$(find -name $s.asd -type f -print -quit)\"" + local asd="\"$(find -name ''${s%%/*}.asd -type f -print -quit)\"" + asds["$asd"]=1 done - echo "(:source-registry-cache ''${asds[@]})" > .cl-source-registry.cache + echo "(:source-registry-cache ''${!asds[@]})" > .cl-source-registry.cache '' + args.postPatch or ""; }; patches = []; From 66a2d20e205f4d53a51c11a250fef8e44e58ab88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kasper=20Ga=C5=82kowski?= Date: Sun, 14 Jul 2024 23:08:42 +0200 Subject: [PATCH 06/22] =?UTF-8?q?ju=C5=BC=20nie=20trzeba=20usuwa=C4=87=20a?= =?UTF-8?q?sd=20bo=20u=C5=BCywam=20cl=20source=20registry=20cache?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nix-cl.nix | 4 ---- 1 file changed, 4 deletions(-) diff --git a/nix-cl.nix b/nix-cl.nix index 79cf862..459dece 100644 --- a/nix-cl.nix +++ b/nix-cl.nix @@ -183,10 +183,6 @@ let asdf = "${asdfFasl}"; }; - postPatch = '' - find . -type f -name '*.asd' -and -not -name $pname.asd -delete - ''; - LISP = "${pkg}/bin/${program} ${flags}"; # Build from $src so that go-to-definition works in SLIME/Sly. From d5b963cd476130ea94660061b06833d09c6b8866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kasper=20Ga=C5=82kowski?= Date: Tue, 16 Jul 2024 02:15:45 +0200 Subject: [PATCH 07/22] TODO build-time-only dependencies --- todo.org | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/todo.org b/todo.org index 75aac58..19bb163 100644 --- a/todo.org +++ b/todo.org @@ -134,3 +134,8 @@ output-files. (When the go-to-definition issue is handled by mass replace of wrapLisp looks unconventional in all-packages.nix. Emacs adds a pkgs attribute to passthru. Let's take another look at such a possibility for common lisps. + +* TODO build-time-only dependencies + +There is at least one system (cl-unicode) which only needs flexi-streams during +build, but not on run-time. How to reflect that using ASDF? From f917b7ba602013f53b274299da7b4af82ccac834 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kasper=20Ga=C5=82kowski?= Date: Tue, 16 Jul 2024 02:21:10 +0200 Subject: [PATCH 08/22] =?UTF-8?q?naprawi=C5=82em=20build=20cl-unicode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Powinienem tak naprawdę użyć output-files w perform, ale patch byłby większy. Powinno to zostać naprawione u źródła tak czy siak. --- packages.nix | 1 + ...enerated-sources-output-translations.patch | 48 +++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 patches/cl-unicode-generated-sources-output-translations.patch diff --git a/packages.nix b/packages.nix index 52c983f..406c1a4 100644 --- a/packages.nix +++ b/packages.nix @@ -90,6 +90,7 @@ let url = "https://github.com/edicl/cl-unicode/archive/refs/tags/v0.1.6.tar.gz"; sha256 = "0ykx2s9lqfl74p1px0ik3l2izd1fc9jd1b4ra68s5x34rvjy0hza"; }; + patches = [ ./patches/cl-unicode-generated-sources-output-translations.patch ]; systems = [ "cl-unicode" ]; lispLibs = with super; [ cl-ppcre diff --git a/patches/cl-unicode-generated-sources-output-translations.patch b/patches/cl-unicode-generated-sources-output-translations.patch new file mode 100644 index 0000000..8b6ef43 --- /dev/null +++ b/patches/cl-unicode-generated-sources-output-translations.patch @@ -0,0 +1,48 @@ +--- a/build/dump.lisp ++++ b/build/dump.lisp +@@ -105,11 +105,13 @@ + writes to the file denoted by RELATIVE-PATH - a path relative to the + location of this source file. Writes a Lisp header to the files + unless NO-HEADER-P is true." +- `(let ((pathname (merge-pathnames ,relative-path ++ `(let* ((pathname (merge-pathnames ,relative-path + (make-pathname :name nil + :type nil + :version nil +- :defaults *this-file*)))) ++ :defaults *this-file*))) ++ (pathname (asdf:apply-output-translations pathname))) ++ (ensure-directories-exist pathname) + (when *compile-verbose* + (format t "~&;;; Writing source file ~A" (file-namestring pathname)) + (force-output)) +--- a/cl-unicode.asd ++++ b/cl-unicode.asd +@@ -48,6 +48,10 @@ +- :output-files (load-op (o c) (values '("lists.lisp" "hash-tables.lisp" "methods.lisp") t)) ++ :output-files (load-op (o c) '("lists.lisp" "hash-tables.lisp" "methods.lisp")) + :perform (load-op (o c) (symbol-call :cl-unicode '#:create-source-files))) + ++(defclass generated-file (cl-source-file) ()) ++(defmethod component-pathname ((component generated-file)) ++ (apply-output-translations (call-next-method))) ++ + (defsystem :cl-unicode + :version "0.1.6" + :serial t +@@ -55,10 +59,10 @@ + :depends-on (:cl-unicode/base) + :license "BSD-2-Clause" + :components ((:file "conditions") +- (:file "lists") +- (:file "hash-tables") ++ (:generated-file "lists") ++ (:generated-file "hash-tables") + (:file "api") +- (:file "methods") ++ (:generated-file "methods") + (:file "test-functions") + (:file "derived") + (:file "alias")) + +Diff finished. Tue Jul 16 01:48:59 2024 From 94ad43ba06e326035709767bb645d6b7ac2ba985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kasper=20Ga=C5=82kowski?= Date: Tue, 16 Jul 2024 05:10:29 +0200 Subject: [PATCH 09/22] =?UTF-8?q?poprawi=C5=82em=20nazwy=20katalog=C3=B3w?= =?UTF-8?q?=20fasl=20kiedy=20pname=20jest=20inny=20od=20systemu?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- builder.lisp | 5 ++--- nix-cl.nix | 5 +++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/builder.lisp b/builder.lisp index e129ed7..f40b5ad 100644 --- a/builder.lisp +++ b/builder.lisp @@ -16,6 +16,7 @@ (defvar *src* (namestring (uiop:getenv "src"))) (defvar *out* (namestring (uiop:getenv "out"))) (defvar *pname* (format nil "~a/" (uiop:getenv "pname"))) +(defvar *declared-systems* (uiop:split-string (uiop:getenv "systems"))) (setf asdf:*resolve-symlinks* nil) @@ -26,11 +27,9 @@ `(:output-translations :disable-cache ("/nix/store/" "/nix/store/") - (,*src* (,*out* "share" "common-lisp" "fasl" :implementation ,*pname*)) + (,*src* (,*out* "share" "common-lisp" "fasl" :implementation ,(first *declared-systems*))) :inherit-configuration)) -(defvar *declared-systems* (uiop:split-string (uiop:getenv "systems"))) - (defvar *system*) ;; (defmethod asdf/component:around-compile-hook :before ((c asdf:system)) diff --git a/nix-cl.nix b/nix-cl.nix index 459dece..53d4d26 100644 --- a/nix-cl.nix +++ b/nix-cl.nix @@ -219,6 +219,11 @@ let for s in $systems; do ln -s $src $out/share/common-lisp/systems/$s done + local sys=''${systems%% *} + local fasl=$out/share/common-lisp/fasl/* + for s in ''${systems#* }; do + ln -s $fasl/$sys $fasl/$s + done ''; dontPatchShebangs = true; From 41565eadd1a96bdade71880b95c458f270da0163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kasper=20Ga=C5=82kowski?= Date: Tue, 16 Jul 2024 05:26:49 +0200 Subject: [PATCH 10/22] =?UTF-8?q?przepisa=C5=82em=20lispWithPackages=20aby?= =?UTF-8?q?=20u=C5=BCywa=C5=82o=20buildEnv?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tak się wydaje czyściej bo nie są kopiowane envy wygenerowane w setup hooku Zamiast tego source registry/output translations mają tylko po jednym wpisie z odniesienem do zbudowanego enva pythonPackages robi podobnie nie wiem jak zachowa się symlink do asdf przy konfliktach w profilu --- builder.lisp | 4 ++++ nix-cl.nix | 53 +++++++++++++++++++++++++++------------------------- 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/builder.lisp b/builder.lisp index f40b5ad..b3baf7e 100644 --- a/builder.lisp +++ b/builder.lisp @@ -83,5 +83,9 @@ (load-systems) +(ensure-directories-exist + (uiop:subpathname (truename *out*) + (make-pathname :directory `(:relative "share" "common-lisp" "asdf" ,(asdf:implementation-identifier))))) + (with-open-file (done ".lisp-build-done" :direction :output) (write-line "done" done)) diff --git a/nix-cl.nix b/nix-cl.nix index 53d4d26..e339647 100644 --- a/nix-cl.nix +++ b/nix-cl.nix @@ -215,6 +215,7 @@ let fi find $out -name '*.so' -exec ln -s "{}" $out/lib \; + ln -s $asdfFasl $out/share/common-lisp/asdf/* mkdir -pv $out/share/common-lisp/systems/ for s in $systems; do ln -s $src $out/share/common-lisp/systems/$s @@ -288,34 +289,36 @@ let # packages - as argument and returns the list of packages to be # installed # TODO(kasper): assert each package has the same lisp and asdf? - lispWithPackagesInternal = clpkgs: packages: - let first = head (lib.attrValues clpkgs); in - (build-asdf-system { - inherit (first) pkg program flags asdf; - # See dontUnpack in build-asdf-system - src = null; - pname = first.pkg.pname; - version = "with-packages"; - lispLibs = packages clpkgs; - systems = []; - }).overrideAttrs(o: { + lispWithPackagesInternal = spec: clpkgs: packages: + let + lisp = spec.pkg; + env = pkgs.buildEnv { + name = "${lisp.pname}-env"; + paths = packages clpkgs; + }; + in stdenv.mkDerivation { + inherit (lisp) pname; + version = "${lisp.version}+packages"; nativeBuildInputs = [ pkgs.makeBinaryWrapper ]; - installPhase = '' + buildCommand = '' mkdir -pv $out/bin + local systems="$(echo ${env}/share/common-lisp/systems/)" + local asdf="$(echo ${env}/share/common-lisp/asdf/*/*)" + local fasl="$(find ${env}/share/common-lisp/fasl/ -mindepth 1 -maxdepth 1 -type d -print -quit)/" + local jars="$(find ${env}/share/java/ -name '*.jar' -print0 | tr '\0' ':')" makeWrapper \ - ${o.pkg}/bin/${o.program} \ - $out/bin/${o.program} \ - --add-flags "${o.flags}" \ - --set ASDF "${o.asdfFasl}" \ - --prefix CL_SOURCE_REGISTRY : "$CL_SOURCE_REGISTRY" \ - --prefix ASDF_OUTPUT_TRANSLATIONS : "$ASDF_OUTPUT_TRANSLATIONS" \ - --prefix LD_LIBRARY_PATH : "$LD_LIBRARY_PATH" \ - --prefix DYLD_LIBRARY_PATH : "$DYLD_LIBRARY_PATH" \ - --prefix CLASSPATH : "$CLASSPATH" \ - --prefix GI_TYPELIB_PATH : "$GI_TYPELIB_PATH" \ - --prefix PATH : "${makeBinPath (o.propagatedBuildInputs or [])}" + ${lisp}/bin/${spec.program} \ + $out/bin/${spec.program} \ + --add-flags "${spec.flags}" \ + --set ASDF "$asdf" \ + --prefix CL_SOURCE_REGISTRY : "$systems/" \ + --prefix ASDF_OUTPUT_TRANSLATIONS : "$systems:$fasl" \ + --prefix LD_LIBRARY_PATH : ${env}/lib \ + --prefix DYLD_LIBRARY_PATH : ${env}/lib \ + --prefix CLASSPATH : "$jars" \ + --prefix PATH : ${env}/bin ''; - }); + }; makeLisp = lib.makeOverridable ({ packageOverlays ? (self: super: {}), spec }: let @@ -323,7 +326,7 @@ let in spec.pkg // { inherit pkgs; inherit (spec) asdf; - withPackages = lispWithPackagesInternal pkgs; + withPackages = lispWithPackagesInternal spec pkgs; buildASDFSystem = args: build-asdf-system (args // spec); }); From 18b825b42e24c5e5f578b3c4304b921abfa13a5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kasper=20Ga=C5=82kowski?= Date: Tue, 16 Jul 2024 06:29:47 +0200 Subject: [PATCH 11/22] =?UTF-8?q?na=20razie=20nie=20sprawdzam=20niezadekla?= =?UTF-8?q?rowanych=20podsystem=C3=B3w?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- builder.lisp | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/builder.lisp b/builder.lisp index b3baf7e..9809380 100644 --- a/builder.lisp +++ b/builder.lisp @@ -17,6 +17,7 @@ (defvar *out* (namestring (uiop:getenv "out"))) (defvar *pname* (format nil "~a/" (uiop:getenv "pname"))) (defvar *declared-systems* (uiop:split-string (uiop:getenv "systems"))) +(defvar *system*) (setf asdf:*resolve-symlinks* nil) @@ -30,7 +31,6 @@ (,*src* (,*out* "share" "common-lisp" "fasl" :implementation ,(first *declared-systems*))) :inherit-configuration)) -(defvar *system*) ;; (defmethod asdf/component:around-compile-hook :before ((c asdf:system)) ;; (format t "[INFO] Checking... ~A ~A" c *system*) @@ -41,23 +41,23 @@ ;; (format t "[INFO] Checking... ~A ~A ~A ~A~%" o c *system* (asdf:output-files o c))) -(defun out-path-p (path) - (or (uiop:string-prefix-p *pwd* (namestring path)) - (uiop:string-prefix-p *out* (namestring path)))) +;; (defun out-path-p (path) +;; (or (uiop:string-prefix-p *pwd* (namestring path)) +;; (uiop:string-prefix-p *out* (namestring path)))) -(defmethod asdf:perform :before ((o asdf:prepare-op) (c asdf:file-component)) - (let ((path (asdf:component-relative-pathname c)) - (path2 (asdf:component-pathname c))) - (format t "FOOOOOOO: ~A >>> ~A~%" path path2))) +;; (defmethod asdf:perform :before ((o asdf:prepare-op) (c asdf:file-component)) +;; (let ((path (asdf:component-relative-pathname c)) +;; (path2 (asdf:component-pathname c))) +;; (format t "FOOOOOOO: ~A >>> ~A~%" path path2))) -(defmethod asdf:perform :before ((o asdf:compile-op) (c asdf:source-file)) - (let ((out (asdf:output-files o c))) - (format t "[INFO] [OUT] Checking... ~A ~A ~A ~A~%" o c *system* out) - ;; (let ((missing (find-if-not #'uiop:probe-file* out))) - (when (find-if-not #'out-path-p out) - (error "While loading '~A': Undeclared subsystem dependency on '~A' in '~A'." *system* (asdf:component-name (asdf:component-system c)) (asdf:primary-system-name c))) - ;; (uiop:ensure-all-directories-exist out) - )) +;; (defmethod asdf:perform :before ((o asdf:compile-op) (c asdf:source-file)) +;; (let ((out (asdf:output-files o c))) +;; (format t "[INFO] [OUT] Checking... ~A ~A ~A ~A~%" o c *system* out) +;; ;; (let ((missing (find-if-not #'uiop:probe-file* out))) +;; (when (find-if-not #'out-path-p out) +;; (error "While loading '~A': Undeclared subsystem dependency on '~A' in '~A'." *system* (asdf:component-name (asdf:component-system c)) (asdf:primary-system-name c))) +;; ;; (uiop:ensure-all-directories-exist out) +;; )) ;; (defmethod asdf:output-files :around ((o asdf:operation) (c asdf:component)) ;; (loop for file in (call-next-method) From c7c1b466fa5db9b4bd68aaa2fdea8e1a2bcab381 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kasper=20Ga=C5=82kowski?= Date: Tue, 16 Jul 2024 06:30:25 +0200 Subject: [PATCH 12/22] uproszczony library path setup hook --- setup-hook.sh | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/setup-hook.sh b/setup-hook.sh index 15c2b2a..5f1f7f0 100644 --- a/setup-hook.sh +++ b/setup-hook.sh @@ -19,23 +19,9 @@ addAsdfOutputTranslation () { } addLibToLibraryPath () { - if test -z "${LD_LIBRARY_PATH:-}"; then - export LD_LIBRARY_PATH="" - fi - if test -z "${DYLD_LIBRARY_PATH:-}"; then - export DYLD_LIBRARY_PATH="" - fi if test -d "$1/lib"; then - if test -n "$(find "$1/lib" -name '*.so' -print -quit)"; then - if [[ ! "$LD_LIBRARY_PATH" =~ "$1/lib" ]]; then - export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$1/lib" - fi - fi - if test -n "$(find "$1/lib" -name '*.dylib' -print -quit)"; then - if [[ ! "$DYLD_LIBRARY_PATH" =~ "$1/lib" ]]; then - export DYLD_LIBRARY_PATH="${DYLD_LIBRARY_PATH:+$DYLD_LIBRARY_PATH:}$1/lib" - fi - fi + addToSearchPath LD_LIBRARY_PATH "$1/lib" + addToSearchPath DYLD_LIBRARY_PATH "$1/lib" fi } From da546b1d54124ab6d7f7d913633d1e24625c4cd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kasper=20Ga=C5=82kowski?= Date: Tue, 16 Jul 2024 06:32:54 +0200 Subject: [PATCH 13/22] =?UTF-8?q?jednak=20nie=20mo=C5=BCna=20nadpisa=C4=87?= =?UTF-8?q?=20output-file=20=C5=BCeby=20load=20fasli=20dzia=C5=82a=C5=82o?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "The manner in which a source file is distinguished from a compiled file is implementation-dependent." --- nix-cl.nix | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/nix-cl.nix b/nix-cl.nix index e339647..a67b057 100644 --- a/nix-cl.nix +++ b/nix-cl.nix @@ -86,10 +86,13 @@ let version = "${asdf.version}-${pkg.pname}"; dontUnpack = true; dontInstall = true; - buildPhase = '' - ${pkg}/bin/${program} \ - ${flags} < \ - <(echo "(compile-file \"${asdf}\" :output-file \"$out\")") + buildCommand = '' + ${pkg}/bin/${program} ${flags} < Date: Tue, 16 Jul 2024 06:35:39 +0200 Subject: [PATCH 14/22] =?UTF-8?q?Co=20z=20kilkoma=20r=C3=B3=C5=BCynmi=20we?= =?UTF-8?q?rsjami=20ASDF=20w=20profilu=3F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nix-cl.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix-cl.nix b/nix-cl.nix index a67b057..ca5f031 100644 --- a/nix-cl.nix +++ b/nix-cl.nix @@ -86,6 +86,7 @@ let version = "${asdf.version}-${pkg.pname}"; dontUnpack = true; dontInstall = true; + # Co z kilkoma różynmi wersjami ASDF w profilu? buildCommand = '' ${pkg}/bin/${program} ${flags} < Date: Tue, 16 Jul 2024 06:40:47 +0200 Subject: [PATCH 15/22] =?UTF-8?q?nie=20dzia=C5=82a=C5=82o=20z=20nieznanego?= =?UTF-8?q?=20powodu=20w=20substitute=20bez=20nawias=C3=B3w?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nix-cl.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nix-cl.nix b/nix-cl.nix index ca5f031..0923660 100644 --- a/nix-cl.nix +++ b/nix-cl.nix @@ -183,7 +183,8 @@ let asdfFasl = buildAsdf { inherit asdf pkg program flags; }; buildScript = pkgs.runCommand "builder.lisp" { inherit asdfFasl; } '' - substitute ${./builder.lisp} $out --replace @asdf@ $asdfFasl/* + substitute ${./builder.lisp} $out \ + --replace @asdf@ "$(find $asdfFasl -mindepth 1 -print -quit)" ''; LISP = "${pkg}/bin/${program} ${flags}"; From 6bb13a0eaec5acd0c5f8ee051c880167a91e4e73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kasper=20Ga=C5=82kowski?= Date: Tue, 16 Jul 2024 06:42:45 +0200 Subject: [PATCH 16/22] =?UTF-8?q?usun=C4=85=C5=82em=20nie=20u=C5=BCywany?= =?UTF-8?q?=20ju=C5=BC=20regexp?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nix-cl.nix | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/nix-cl.nix b/nix-cl.nix index 0923660..c17ea0b 100644 --- a/nix-cl.nix +++ b/nix-cl.nix @@ -198,20 +198,7 @@ let ''; # Copy compiled files to store - # - # Make sure to include '$' in regex to prevent skipping - # stuff like 'iolib.asdf.asd' for system 'iolib.asd' - # - # Same with '/': `local-time.asd` for system `cl-postgres+local-time.asd` - - # TODO share source between implementations in nix profile - # around method for prepare-op on source-file? - installPhase = - let - mkSystemsRegex = systems: - concatMapStringsSep "\\|" (replaceStrings ["." "+"] ["[.]" "[+]"]) systems; - in - '' + installPhase = '' # TODO this is a hack at best. Fix osicat and others using cffi-grovel # by setting a different output translation for shared objects if [ -n "$(find $out -name '*.so' -print -quit)" ]; then From a3c9f8cee79746d52648741cecca793fa38267f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kasper=20Ga=C5=82kowski?= Date: Tue, 16 Jul 2024 06:44:32 +0200 Subject: [PATCH 17/22] =?UTF-8?q?og=C3=B3lnie=20teraz=20dzia=C5=82a=20budo?= =?UTF-8?q?wanie=20tylko=20na=20sbcl?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit inne impl mają problem z widocznością output translations prawdopodobnie przyczyną jest sposób traktowania symlinków From c5cbfc86542dad1ee9284815405199c3767f5ee3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kasper=20Ga=C5=82kowski?= Date: Tue, 16 Jul 2024 06:46:52 +0200 Subject: [PATCH 18/22] dodatkowa notatka do asdf:load-system MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit z powodu użycia symlinków w share/common-lisp, aby działało wczytywanie load-system wymagane jest ustawienie asdf:*resolve-symlinks* na nil. Inaczej będzie mapował do zlinkowanego źródła (/nix/store/...-source/*.fasl) zamiast do ścieżki logicznej (~/.nix-profile/share/common-lisp/fasl/.../*.fasl). From e0515beffc80ab219ff2534980c05af1826cbb2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kasper=20Ga=C5=82kowski?= Date: Tue, 16 Jul 2024 18:59:26 +0200 Subject: [PATCH 19/22] =?UTF-8?q?poprawione=20tworzenie=20podw=C3=B3jnych?= =?UTF-8?q?=20link=C3=B3w=20do=20pierwszego=20systemu?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nix-cl.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nix-cl.nix b/nix-cl.nix index c17ea0b..e83b907 100644 --- a/nix-cl.nix +++ b/nix-cl.nix @@ -212,7 +212,8 @@ let ln -s $src $out/share/common-lisp/systems/$s done local sys=''${systems%% *} - local fasl=$out/share/common-lisp/fasl/* + local fasl="$(find "$out/share/common-lisp/fasl/" -mindepth 1 -maxdepth 1 -type d -print -quit)" + systems="$systems " for s in ''${systems#* }; do ln -s $fasl/$sys $fasl/$s done From 4bae9fc130ff298f59d5eb226157be7b07e7e084 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kasper=20Ga=C5=82kowski?= Date: Tue, 16 Jul 2024 20:01:57 +0200 Subject: [PATCH 20/22] =?UTF-8?q?Podej=C5=9Bcie=20z=20symlinkami=20+=20:TR?= =?UTF-8?q?EE=20nie=20b=C4=99dzie=20dzia=C5=82a=C4=87.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Różne implementacje mają niekompatybilne traktowanie symlinków. SBCL potrafi nie rozwiązywać. ECL rozwiązuje nawet przy :resolve-symlinks nil jeśli katalog nadrzędny jest symlinkiem. ABCL również potrafi nie rozwiązywać ale wymaga to patcha w ASDF. Ogólnie robi się gnój i chyba lepiej będzie wrócić do niestety bardziej skomplikowanego (ale dającego więcej kontroli) podejścia z :INCLUDE plików konfiguracyjnych w podkatalogach src/fasl z komponentem :IMPLEMENTATION. From ce94130005ac79235a1308acf931352fc2570deb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kasper=20Ga=C5=82kowski?= Date: Tue, 16 Jul 2024 20:10:10 +0200 Subject: [PATCH 21/22] =?UTF-8?q?translation-function=20te=C5=BC=20raczej?= =?UTF-8?q?=20nie=20zadzia=C5=82a?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ponieważ symlinki są rozwiązane już przez cl:directory i dalej nic z tym nie zrobię '#' will be ignored, and an empty message aborts the commit. # # On branch scratch # Your branch is up to date with 'origin/scratch'. # From f42735214e46c3ccc001cc845d8eb6552962bc3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kasper=20Ga=C5=82kowski?= Date: Thu, 18 Jul 2024 00:44:28 +0200 Subject: [PATCH 22/22] =?UTF-8?q?Spr=C3=B3bowa=C5=82em=20zapisywania=20out?= =?UTF-8?q?put=20translations=20do=20conf.d=20poprzedzonych=20#+ECL,=20?= =?UTF-8?q?=C5=BCeby=20nie=20by=C5=82o=20potrzeby=20tworzenia=20podkatalog?= =?UTF-8?q?=C3=B3w=20na=20ka=C5=BCd=C4=85=20implementacj=C4=99.=20W=20ten?= =?UTF-8?q?=20spos=C3=B3b=20dzia=C5=82a=C5=82oby=20to=20przy=20wrzuceniu?= =?UTF-8?q?=20ca=C5=82ego=20conf.d=20do=20XDG=5FDATA=5FDIRS.=20Niestety=20?= =?UTF-8?q?w=20ten=20spos=C3=B3b=20nie=20zostanie=20obs=C5=82u=C5=BCony=20?= =?UTF-8?q?przypadek=20r=C3=B3=C5=BCnych=20wersji=20tej=20samej=20implemen?= =?UTF-8?q?tacji,=20wi=C4=99c=20prawdopodobnie=20b=C4=99d=C4=99=20musia?= =?UTF-8?q?=C5=82=20tak=20czy=20siak=20tworzy=C4=87=20podkatalogi=20asdf:i?= =?UTF-8?q?mplementation-identifier=20i=20by=C4=87=20mo=C5=BCe=20doda?= =?UTF-8?q?=C4=87=20now=C4=85=20zmienn=C4=85=20NIX=5FASDF=5FOUTPUT=5FTRANS?= =?UTF-8?q?LATION=5FCONF=5FDIRS=20po=20to,=20=C5=BCeby=20nie=20musie=C4=87?= =?UTF-8?q?=20wrzuca=C4=87=20sexpr=20do=20ASDF=5FOUTPUT=5FTRANSLATIONS=20c?= =?UTF-8?q?o=20przeszkadza=20w=20ustawianiu=20jej=20z=20u=C5=BCyciem=20she?= =?UTF-8?q?ll-friendly=20DSL=20przez=20u=C5=BCytkownika.=20Ta=20nowa=20zmi?= =?UTF-8?q?enna=20dzia=C5=82a=C5=82aby=20tylko=20w=20wrapperze=20withPacka?= =?UTF-8?q?ges=20(lub=20by=C4=87=20mo=C5=BCe=20r=C3=B3wnie=C5=BC=20w=20bui?= =?UTF-8?q?ldPhase=20po=20setup=20hooku),=20wi=C4=99c=20raczej=20nie=20mus?= =?UTF-8?q?z=C4=99=20si=C4=99=20przejmowa=C4=87=20u=C5=BCywalno=C5=9Bci?= =?UTF-8?q?=C4=85=20tego=20przez=20u=C5=BCytkownika,=20tzn.=20ta=20zmienna?= =?UTF-8?q?=20b=C4=99dzie=20szczeg=C3=B3=C5=82em=20implementacyjnym=20nie?= =?UTF-8?q?=20udokumentowanym=20w=20podr=C4=99czniku.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- asdf-nix.patch | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/asdf-nix.patch b/asdf-nix.patch index 8e8d2c6..7d9f3c4 100644 --- a/asdf-nix.patch +++ b/asdf-nix.patch @@ -1,5 +1,15 @@ --- a/output-translations.lisp +++ b/output-translations.lisp +@@ -74,7 +74,7 @@ + (and (consp directive) + (or (and (length=n-p directive 2) + (or (and (eq (first directive) :include) +- (typep (second directive) '(or string pathname null))) ++ (location-designator-p (second directive))) + (and (location-designator-p (first directive)) + (or (location-designator-p (second directive)) + (location-function-p (second directive)))))) + @@ -146,6 +146,7 @@ '(environment-output-translations user-output-translations-pathname @@ -8,22 +18,31 @@ system-output-translations-pathname system-output-translations-directory-pathname)) -@@ -184,6 +185,14 @@ +@@ -184,6 +185,13 @@ :direction direction)) (defun environment-output-translations () (getenv "ASDF_OUTPUT_TRANSLATIONS")) -+ (defun nix-profile-output-translations () ++ defun nix-profile-output-translations () + `(:output-translations + ,@(loop :for profile :in (reverse (split-string (or (getenv "NIX_PROFILES") ""))) -+ :for share := (subpathname (ensure-directory-pathname profile) "share/") -+ :for sysdir := (namestring (subpathname share "common-lisp/systems/")) -+ :for fasldir := (namestring (subpathname share "common-lisp/fasl/")) -+ :collect `(,sysdir (,fasldir :implementation))) ++ :for share := (subpathname (ensure-directory-pathname profile) "share/common-lisp/") ++ :for confd := (namestring (subpathname share "asdf-output-translations.conf.d/")) ++ :collect `(:include (,confd :implementation))) + :inherit-configuration)) ;;; Processing the configuration. +@@ -209,7 +209,7 @@ + (dst (second directive))) + (if (eq src :include) + (when dst +- (process-output-translations (pathname dst) :inherit nil :collect collect)) ++ (process-output-translations (resolve-location dst :ensure-directory t) :inherit nil :collect collect)) + (when src + (let ((trusrc (or (eql src t) + (let ((loc (resolve-location src :ensure-directory t :wilden t))) + --- a/source-registry.lisp +++ b/source-registry.lisp @@ -187,6 +187,7 @@ @@ -34,7 +53,7 @@ system-source-registry system-source-registry-directory default-system-source-registry) -@@ -227,5 +228,11 @@ +@@ -227,6 +228,12 @@ :direction direction)) (defun environment-source-registry () (getenv "CL_SOURCE_REGISTRY")) @@ -46,4 +65,5 @@ + :inherit-configuration)) - ;;; Process the source-registry configuration \ No newline at end of file + ;;; Process the source-registry configuration +