Adding zpm "ci" command to install from a lock file

CHANGELOG.md

@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [0.10.8] - Unreleased
 
+### Added
+- #962: Adding zpm -from-lockfile flag to install/load from a lock file
+
 ### Fixed
 - #1192: Fix dependency resolution dropping incompatibilities and along with that fix a bug where
 semver Intersection would incorrectly pass if an exact version is compared against a range even with

src/cls/IPM/General/LockFile.cls

@@ -69,12 +69,12 @@ ClassMethod CreateLockFileForModule(
         $$$ThrowOnError(lockFile.Dependencies.SetAt(dependencyVal, mod.Name))
 
         // Add the dependency's repository to the lock file
-        do AddRepositoryToLockFile(.lockFile, mod.Repository, verbose)
+        do ..AddRepositoryToLockFile(.lockFile, mod.Repository, verbose)
     }
     // Add repository for base module if not already added by a dependency
     // Skip undefined repositories as that means the module was installed via the zpm "load" command
     if (module.Repository '= "") {
-        do AddRepositoryToLockFile(.lockFile, module.Repository, verbose)
+        do ..AddRepositoryToLockFile(.lockFile, module.Repository, verbose)
     }
 
     $$$ThrowOnError(lockFile.%JSONExportToStream(.lockFileJSON, "LockFileMapping"))
@@ -116,4 +116,93 @@ ClassMethod GetRepo(repoName As %String) As %IPM.Repo.Definition [ Internal ]
     }
 }
 
+ClassMethod InstallFromLockFile(
+	directory As %String,
+	ByRef params)
+{
+    set verbose = $get(params("Verbose"), 0)
+
+    set lockFilePath = ##class(%File).NormalizeFilename("module-lock.json", directory)
+    try {
+        set lockFileJSON = ##class(%DynamicObject).%FromJSONFile(lockFilePath)
+    } catch {
+        $$$ThrowStatus($$$ERROR($$$GeneralError,$$$FormatText("Unable to parse lock file JSON at path: %1", lockFilePath)))
+    }
+
+    set repositories = lockFileJSON.%Get("repositories", {})
+    set dependencies = lockFileJSON.%Get("dependencies", {})
+
+    // Install repositories (if they don't already exist)
+    set repoIter = repositories.%GetIterator()
+    while repoIter.%GetNext(.repoName, .repoVals) {
+        if ##class(%IPM.Repo.Definition).ServerDefinitionKeyExists(repoName) {
+            if (verbose) {
+                write !, "Repo: "_repoName_" already exists, skipping creating new one from lock file"
+            }
+            continue
+        } elseif (verbose) {
+            write !, "Creating repo: "_repoName_" from lock file"
+        }
+        do ##class(%IPM.Repo.Definition).CollectServerTypes(.types)
+        set repoClass = types(repoVals.type)
+        set repoVals.name = repoName
+        do $classmethod(repoClass, "LockFileValuesToModifiers", repoVals, .modifiers)
+        $$$ThrowOnError($classmethod(repoClass,"Configure",0,.modifiers,.tData,repoClass))
+    }
+
+    // Install modules (even if they already exist)
+    do ..GetOrderedDependenciesList(dependencies, .orderedDependenciesList)
+    set depName = ""
+    for i=1:1:$listlength(orderedDependenciesList) {
+        set depName = $list(orderedDependenciesList, i)
+        set depVals = dependencies.%Get(depName)
+
+        set version = depVals.version
+        set repository = depVals.repository
+
+        // Call Install() on dependency modules but set flag "LockFileInstallStarted" so we don't try installing from the dependency module's lock file
+        set commandInfo = "install"
+        set commandInfo("data", "Verbose") = verbose
+        set commandInfo("parameters","module") = repository_"/"_depName
+        set commandInfo("parameters", "version") = version
+        set commandInfo("data", "LockFileInstallStarted") = 1
+        set commandInfo("data","FromLockFile") = 1
+
+        do ##class(%IPM.Main).Install(.commandInfo)
+    }
+}
+
+/// The dependencies list in a lock file is listed alphabetically.
+/// This method compiles the dependencies and creates an ordered list such that no module is listed
+/// before one of its dependencies. Can then trace the list this outputs and install in order
+ClassMethod GetOrderedDependenciesList(
+	dependencies As %DynamicObject,
+	ByRef orderedDependenciesList As %List = "") [ Internal ]
+{
+    set depIter = dependencies.%GetIterator()
+    while depIter.%GetNext(.depName, .depVals) {
+        do ..AddToOrderedDependenciesList(depName, dependencies, .orderedDependenciesList)
+    }
+}
+
+/// For a dependency, recursively adds all dependencies to the ordered list, followed by this dependency
+ClassMethod AddToOrderedDependenciesList(
+	dependencyName As %String,
+	dependencies As %DynamicObject,
+	ByRef orderedDependenciesList As %List) [ Internal, Private ]
+{
+    set depVals = dependencies.%Get(dependencyName)
+    set transientDeps = depVals.%Get("dependencies", {})
+    set transientIter = transientDeps.%GetIterator()
+    while transientIter.%GetNext(.transDepName) {
+        // If dependency hasn't been installed yet, then recursively run this method on it
+        if '$listfind(orderedDependenciesList, transDepName) {
+            do ..AddToOrderedDependenciesList(transDepName, dependencies, .orderedDependenciesList)
+        }
+    }
+    if '$listfind(orderedDependenciesList, dependencyName) {
+        set orderedDependenciesList = orderedDependenciesList _ $listbuild(dependencyName)
+    }
+}
+
 }

src/cls/IPM/Lifecycle/Base.cls

@@ -1344,6 +1344,7 @@ Method %Export(
             "changelog.md",
             "license",
             "requirements.txt",
+            "module-lock.json",
         )
         set tRes = ##class(%File).FileSetFunc(..Module.Root)
         while tRes.%Next() {

src/cls/IPM/Main.cls

@@ -296,7 +296,8 @@ load C:\module\root\path -env C:\path\to\env1.json;C:\path\to\env2.json
 <modifier name="extra-pip-flags" dataAlias="ExtraPipFlags" value="true" description="Extra flags to pass to pip when installing python dependencies. Surround the flags (and values) with quotes if spaces are present. Default flags are &quot;--target &lt;target&gt; --python-version &lt;pyversion&gt; --only-binary=:all:&quot;." />
 <modifier name="synchronous" value="false" deprecated="true" description="DEPRECATED. Dependencies are now always loaded synchronously with independent lifecycle phases doing their own multi-threading as needed." />
 <modifier name="force" aliases="f" value="false" description="Allows the user to load a newer version of an existing module without running update steps." />
-<modifier name="create-lockfile" aliases="lock" dataAlias="CreateLockFile" dataValue="1" description="Upon load, creates/updates the module's lock file." />
+<modifier name="create-lockfile" dataAlias="CreateLockFile" dataValue="1" description="Upon load, creates/updates the module's lock file." />
+<modifier name="from-lockfile" dataAlias="FromLockFile" dataValue="1" description="Load the module from the lock file present at the path." />
 
 <!-- Parameters -->
 <parameter name="path" required="true" description="Directory on the local filesystem, containing a file named module.xml" />
@@ -423,7 +424,8 @@ install -env /path/to/env1.json;/path/to/env2.json example-package
 <modifier name="extra-pip-flags" dataAlias="ExtraPipFlags" value="true" description="Extra flags to pass to pip when installing python dependencies. Surround the flags (and values) with quotes if spaces are present. Default flags are &quot;--target &lt;target&gt; --python-version &lt;pyversion&gt; --only-binary=:all:&quot;."/>
 <modifier name="synchronous" value="false" deprecated="true" description="DEPRECATED. Dependencies are now always loaded synchronously with independent lifecycle phases doing their own multi-threading as needed." />
 <modifier name="force" aliases="f" value="false" description="Allows the user to install a newer version of an existing module without running update steps." />
-<modifier name="create-lockfile" aliases="lock" dataAlias="CreateLockFile" dataValue="1" description="Upon install, creates/updates the module's lock file." />
+<modifier name="create-lockfile" dataAlias="CreateLockFile" dataValue="1" description="Upon install, creates/updates the module's lock file." />
+<modifier name="from-lockfile" dataAlias="FromLockFile" dataValue="1" description="Load the module from the lock file present at the path." />
 
 </command>
 
@@ -1096,8 +1098,8 @@ ClassMethod ShellInternal(
                 do ..ModuleVersion(.tCommandInfo)
             } elseif (tCommandInfo = "information") {
                 do ..Information(.tCommandInfo)
-			} elseif (tCommandInfo = "history") {
-				do ..History(.tCommandInfo)
+            } elseif (tCommandInfo = "history") {
+                do ..History(.tCommandInfo)
             }
         } catch pException {
             if (pException.Code = $$$ERCTRLC) {
@@ -2394,36 +2396,40 @@ ClassMethod LoadInternal(
         }
     }
 
-    // When loading a module from a local folder, there might be a <mod root>/.modules/ folder containing dependencies.
-    // It's easier to configure a temporary repository than to handle this case in the dependency resolution code.
     set tTargetDirectory = $get(tTargetDirectory, tDirectoryName)
-
-    // For tarball and URL sources, copy the extracted content to <ModuleRoot>/<name>/<version>/
-    // and clean up the temp extraction directory. Directory loads are left in-place so that
-    // live-edit workflows (mounted volumes, bootstrap) continue to work without re-loading.
-    if $get(tTempExtractionRoot) '= "" {
-        set tModuleObj = ##class(%IPM.Utils.Module).GetModuleObjectFromPath(tTargetDirectory, .tFound)
-        if 'tFound {
-            $$$ThrowStatus($$$ERROR($$$GeneralError,"No Module element found in module.xml at "_tTargetDirectory))
-        }
-        set tFinalDirectory = ##class(%Library.File).NormalizeDirectory(
-            ##class(%IPM.Repo.UniversalSettings).GetModuleRoot() _ tModuleObj.Name _ "/" _ tModuleObj.VersionString)
-        if ##class(%File).DirectoryExists(tFinalDirectory) {
-            $$$ThrowOnError(##class(%IPM.Utils.File).RemoveDirectoryTree(tFinalDirectory))
-        }
-        $$$ThrowOnError(##class(%IPM.Utils.File).CreateDirectoryChain(tFinalDirectory))
-        // Destination was just created empty above, so pDeleteFirst=0 is intentional.
-        // If CopyDir fails partway through, tFinalDirectory may be left in a partial state.
-        $$$ThrowOnError(##class(%IPM.Utils.File).CopyDir(tTargetDirectory, tFinalDirectory, 0))
-        $$$ThrowOnError(##class(%IPM.Utils.File).RemoveDirectoryTree(tTempExtractionRoot))
-        set tTargetDirectory = tFinalDirectory
-    }
-
-    set dotModules = ##class(%File).NormalizeDirectory(".modules", tTargetDirectory)
-    set tmpRepoMgr = ##class(%IPM.General.TempLocalRepoManager).%New(dotModules, 1)
-    set tSC = ##class(%IPM.Utils.Module).LoadNewModule(tTargetDirectory, .tParams, , pLog)
-    set tSC = $$$ADDSC(tSC, tmpRepoMgr.CleanUp())
-    $$$ThrowOnError(tSC)
+    if ($get(pCommandInfo("data", "FromLockFile"))) {
+        do ##class(%IPM.General.LockFile).InstallFromLockFile(tTargetDirectory, .tParams)
+    } else {
+        // When loading a module from a local folder, there might be a <mod root>/.modules/ folder containing dependencies.
+        // It's easier to configure a temporary repository than to handle this case in the dependency resolution code.
+
+        // For tarball and URL sources, copy the extracted content to <ModuleRoot>/<name>/<version>/
+        // and clean up the temp extraction directory. Directory loads are left in-place so that
+        // live-edit workflows (mounted volumes, bootstrap) continue to work without re-loading.
+        if $get(tTempExtractionRoot) '= "" {
+            set tModuleObj = ##class(%IPM.Utils.Module).GetModuleObjectFromPath(tTargetDirectory, .tFound)
+            if 'tFound {
+                $$$ThrowStatus($$$ERROR($$$GeneralError,"No Module element found in module.xml at "_tTargetDirectory))
+            }
+            set tFinalDirectory = ##class(%Library.File).NormalizeDirectory(
+                ##class(%IPM.Repo.UniversalSettings).GetModuleRoot() _ tModuleObj.Name _ "/" _ tModuleObj.VersionString)
+            if ##class(%File).DirectoryExists(tFinalDirectory) {
+                $$$ThrowOnError(##class(%IPM.Utils.File).RemoveDirectoryTree(tFinalDirectory))
+            }
+            $$$ThrowOnError(##class(%IPM.Utils.File).CreateDirectoryChain(tFinalDirectory))
+            // Destination was just created empty above, so pDeleteFirst=0 is intentional.
+            // If CopyDir fails partway through, tFinalDirectory may be left in a partial state.
+            $$$ThrowOnError(##class(%IPM.Utils.File).CopyDir(tTargetDirectory, tFinalDirectory, 0))
+            $$$ThrowOnError(##class(%IPM.Utils.File).RemoveDirectoryTree(tTempExtractionRoot))
+            set tTargetDirectory = tFinalDirectory
+        }
+
+        set dotModules = ##class(%File).NormalizeDirectory(".modules", tTargetDirectory)
+        set tmpRepoMgr = ##class(%IPM.General.TempLocalRepoManager).%New(dotModules, 1)
+        set tSC = ##class(%IPM.Utils.Module).LoadNewModule(tTargetDirectory, .tParams, , pLog)
+        set tSC = $$$ADDSC(tSC, tmpRepoMgr.CleanUp())
+        $$$ThrowOnError(tSC)
+    }
 }
 
 ClassMethod CheckModuleNamespace() As %Status
@@ -2530,12 +2536,11 @@ ClassMethod Install(
                         $$$ThrowStatus($$$ERROR($$$GeneralError, "Deployed package '" _ tModuleName _ "' " _ tResult.VersionString _ " not supported on this platform " _ platformVersion _ "."))
                     }
                 }
-				$$$ThrowOnError(log.SetSource(tResult.ServerName))
-				$$$ThrowOnError(log.SetVersion(tResult.Version))
+                $$$ThrowOnError(log.SetSource(tResult.ServerName))
+                $$$ThrowOnError(log.SetVersion(tResult.Version))
                 $$$ThrowOnError(##class(%IPM.Utils.Module).LoadQualifiedReference(tResult, .tParams, , log))
             }
         } else {
-            set tPrefix = ""
             if (tModuleName '= "") {
                 if (tVersion '= "") {
                     $$$ThrowStatus($$$ERROR($$$GeneralError, tModuleName_" "_tVersion_" not found in any repository."))
@@ -2549,10 +2554,10 @@ ClassMethod Install(
             }
         }
     } catch ex {
-		$$$ThrowOnError(log.Finalize(ex.AsStatus(), devMode))
-		throw ex
-	}
-	$$$ThrowOnError(log.Finalize($$$OK, devMode))
+        $$$ThrowOnError(log.Finalize(ex.AsStatus(), devMode))
+        throw ex
+    }
+    $$$ThrowOnError(log.Finalize($$$OK, devMode))
 }
 
 ClassMethod Reinstall(ByRef pCommandInfo) [ Internal ]

src/cls/IPM/Repo/Definition.cls

@@ -299,6 +299,16 @@ Method LockFileTypeGet()
     return ..#MONIKER
 }
 
+/// We use the "LockFileMapping" XData block to export a repo definition to a lock file.
+/// When importing from a lock file, doing an import and save using the same XData block won't work.
+/// Instead, we must create a set of modifiers and call %IPM.Repo.Definition::Configure()
+/// This method accepts the JSON repository values from a lock file and populates a modifers object to then call Configure() with
+ClassMethod LockFileValuesToModifiers(
+	lockFileValues As %DynamicObject,
+	Output modifiers) [ Abstract ]
+{
+}
+
 Storage Default
 {
 <Data name="RepoDefinitionDefaultData">

src/cls/IPM/Repo/Filesystem/Definition.cls

@@ -357,11 +357,21 @@ ClassMethod ScanDirectory(
     quit tSC
 }
 
+ClassMethod LockFileValuesToModifiers(
+	lockFileValues As %DynamicObject,
+	Output modifiers)
+{
+    set modifiers("filesystem") = ""
+    set modifiers("name") = lockFileValues.%Get("name")
+    set modifiers("path") = lockFileValues.%Get("root")
+    set modifiers("depth") = lockFileValues.%Get("depth")
+    set modifiers("read-only") = lockFileValues.%Get("readOnly")
+}
+
 XData LockFileMapping
 {
 <Mapping xmlns="http://www.intersystems.com/jsonmapping">
   <Property Name="LockFileType" FieldName="type" />
-  <Property Name="OverriddenSortOrder" FieldName="overriddenSortOrder" />
   <Property Name="ReadOnly" FieldName="readOnly" />
   <Property Name="Root" FieldName="root" />
   <Property Name="Depth" FieldName="depth" />

src/cls/IPM/Repo/Oras/Definition.cls

@@ -154,11 +154,34 @@ Method GetPublishingManager(ByRef status)
     return ##class(%IPM.Repo.Oras.PublishManager).%Get(.status)
 }
 
+ClassMethod LockFileValuesToModifiers(
+	lockFileValues As %DynamicObject,
+	Output modifiers)
+{
+    set modifiers("oras") = ""
+    set modifiers("name") = lockFileValues.%Get("name")
+    set modifiers("read-only") = lockFileValues.%Get("readOnly")
+    set modifiers("url") = lockFileValues.%Get("url")
+    set modifiers("namespace") = lockFileValues.%Get("orasNamespace")
+
+    // The following variables are set as system level variables for us to get
+    // Naming convention for those follow the name of the repository, first converting any '-' to '_',
+    // then removing everything except for alphabetic characters, numbers, and '_' to use as variable prefix
+    // lastly, adds a suffix based on the modifier
+    // Examples: <repo name>        - <prefix>
+    //           "registry"         - "registry"
+    //           "ORAS!Repo(5)?"    - "ORASRepo5"
+    //           "My-Repository-2"  - "My_Repository_2"
+    set prefix = $zstrip($replace(modifiers("name"), "-", "_"), "*E'N'A")
+    set modifiers("username") = $system.Util.GetEnviron(prefix_"_username")
+    set modifiers("password") = $system.Util.GetEnviron(prefix_"_password")
+    set modifiers("token") = $system.Util.GetEnviron(prefix_"_token")
+}
+
 XData LockFileMapping
 {
 <Mapping xmlns="http://www.intersystems.com/jsonmapping">
   <Property Name="LockFileType" FieldName="type" />
-  <Property Name="OverriddenSortOrder" FieldName="overriddenSortOrder" />
   <Property Name="ReadOnly" FieldName="readOnly" />
   <Property Name="URL" FieldName="url" />
   <Property Name="Namespace" FieldName="orasNamespace" />

src/cls/IPM/Repo/Remote/Definition.cls

@@ -132,6 +132,29 @@ Method GetPublishingManager(ByRef status)
     return ##class(%IPM.Repo.Remote.PublishManager).%Get(.status)
 }
 
+ClassMethod LockFileValuesToModifiers(
+	lockFileValues As %DynamicObject,
+	Output modifiers)
+{
+    set modifiers("remote") = ""
+    set modifiers("name") = lockFileValues.%Get("name")
+    set modifiers("url") = lockFileValues.%Get("url")
+    set modifiers("read-only") = lockFileValues.%Get("readOnly")
+
+    // The following variables are set as system level variables for us to get
+    // Naming convention for those follow the name of the repository, first converting any '-' to '_',
+    // then removing everything except for alphabetic characters, numbers, and '_' to use as variable prefix
+    // lastly, adds a suffix based on the modifier
+    // Examples: <repo name>        - <prefix>
+    //           "registry"         - "registry"
+    //           "ORAS!Repo(5)?"    - "ORASRepo5"
+    //           "My-Repository-2"  - "My_Repository_2"
+    set prefix = $zstrip($replace(modifiers("name"), "-", "_"), "*E'N'A")
+    set modifiers("username") = $system.Util.GetEnviron(prefix_"_username")
+    set modifiers("password") = $system.Util.GetEnviron(prefix_"_password")
+    set modifiers("token") = $system.Util.GetEnviron(prefix_"_token")
+}
+
 Method LockFileTypeGet()
 {
     return ..#MONIKERALIAS
@@ -141,7 +164,6 @@ XData LockFileMapping
 {
 <Mapping xmlns="http://www.intersystems.com/jsonmapping">
   <Property Name="LockFileType" FieldName="type" />
-  <Property Name="OverriddenSortOrder" FieldName="overriddenSortOrder" />
   <Property Name="ReadOnly" FieldName="readOnly" />
   <Property Name="URL" FieldName="url" />
 </Mapping>