nuget downloader done

go.mod

@@ -4,5 +4,7 @@ go 1.16
 
 require (
 	github.com/antchfx/xmlquery v1.3.5
+	github.com/artdarek/go-unzip v1.0.0 // indirect
 	github.com/mcuadros/go-version v0.0.0-20190830083331-035f6764e8d2
+	github.com/xyproto/unzip v0.0.0-20150601123358-823950573952 // indirect
 )

go.sum

@@ -2,10 +2,14 @@ github.com/antchfx/xmlquery v1.3.5 h1:I7TuBRqsnfFuL11ruavGm911Awx9IqSdiU6W/ztSmV
 github.com/antchfx/xmlquery v1.3.5/go.mod h1:64w0Xesg2sTaawIdNqMB+7qaW/bSqkQm+ssPaCMWNnc=
 github.com/antchfx/xpath v1.1.10 h1:cJ0pOvEdN/WvYXxvRrzQH9x5QWKpzHacYO8qzCcDYAg=
 github.com/antchfx/xpath v1.1.10/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
+github.com/artdarek/go-unzip v1.0.0 h1:Ja9wfhiXyl67z5JT37rWjTSb62KXDP+9jHRkdSREUvg=
+github.com/artdarek/go-unzip v1.0.0/go.mod h1:KhX4LV7e4UwWCTo7orBYnJ6LJ/dZTI6jXxUg69hO/C8=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/mcuadros/go-version v0.0.0-20190830083331-035f6764e8d2 h1:YocNLcTBdEdvY3iDK6jfWXvEaM5OCKkjxPKoJRdB3Gg=
 github.com/mcuadros/go-version v0.0.0-20190830083331-035f6764e8d2/go.mod h1:76rfSfYPWj01Z85hUf/ituArm797mNKcvINh1OlsZKo=
+github.com/xyproto/unzip v0.0.0-20150601123358-823950573952 h1:pCgYs8AOzRpEmdM55hrSnd+TZOr+1/m9Y9KkFIZPwiU=
+github.com/xyproto/unzip v0.0.0-20150601123358-823950573952/go.mod h1:ygEdojOpwX5uG71ECHji5sod2rpZ+9hqNeAZD50S84Q=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=

tools/nuget-download-package/main.go

 package main
 
 import (
+	"encoding/json"
 	"github.com/antchfx/xmlquery"
+	"github.com/mcuadros/go-version"
 	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
+	"path/filepath"
 	"strings"
 	"sync"
 )
@@ -22,7 +25,7 @@ dotnet add also requires a valid csproj. It's heavy, complex, and slow. It sucks
 
 Let's write a simple one!
 */
-const nugetConfigPath = "~/.nuget/NuGet/NuGet.Config"
+const nugetConfigPath = "/home/recolic/.nuget/NuGet/NuGet.Config"
 
 func panicErrorIfAny(err error, title string) {
 	if err != nil {
@@ -35,16 +38,48 @@ func logErrorIfAny(err error, title string) {
 	}
 }
 
-func pkgVersionLeftIsGreater(left, right string) bool {
-	// return if left > right
-	return true
+
+func netVersionNugetFormatToStandardFormat(netVersionNugetFormat string) string {
+	// nuget format: ".NETFramework4.6.2", ".NETCoreApp2.0", ".NETStandard2.0"
+	// standard format: "net462", "netcoreapp2.0", "net5.0", "netstandard2.1"
+	// https://docs.microsoft.com/en-us/dotnet/standard/frameworks
+	netVersionNugetFormat = strings.ToLower(netVersionNugetFormat)
+	if strings.HasPrefix(netVersionNugetFormat, ".netframework") {
+		frameworkVer := netVersionNugetFormat[len(".netframework"):]
+		return "net" + strings.ReplaceAll(frameworkVer, ".", "")
+	} else if strings.HasPrefix(netVersionNugetFormat, ".netcoreapp") {
+		return netVersionNugetFormat[1:]
+	} else if strings.HasPrefix(netVersionNugetFormat, ".netstandard") {
+		return netVersionNugetFormat[1:]
+	} else {
+		panic("Unknown nuget-format net version: " + netVersionNugetFormat)
+	}
 }
 
-var nugetIndexJsonCache []string
-func initNugetIndexJsonCache(pkgName string) {
-	// download index.json from every source, and cache them in global variable.
-	nugetIndexJsonCache = []string{}
+func httpGet(url, authu, authp, outputFilepath string) string {
+	// This function would follow HTTP 30x
+	log.Println("HTTP GET " + url + " with username " + authu)
+	req, err := http.NewRequest("GET", url, nil)
+	panicErrorIfAny(err, "http.NewRequest")
+	if authu + authp != "" {
+		req.SetBasicAuth(authu, authp)
+	}
+	resp, err := http.DefaultClient.Do(req)
+	logErrorIfAny(err, "GET request")
+	defer resp.Body.Close()
+	respTxt, err := ioutil.ReadAll(resp.Body)
+	logErrorIfAny(err, "GET request")
+	if outputFilepath != "" {
+		err := ioutil.WriteFile(outputFilepath, respTxt, 0777)
+		logErrorIfAny(err, "WriteFile " + outputFilepath)
+		return ""
+	} else {
+		return string(respTxt)
+	}
+}
 
+func initNugetIndexJsonCache(pkgName string) (nugetIndexJsonCache, nugetIndexJsonCache_Login []string) {
+	// download index.json from every source, and cache them in array.
 	f, err := os.Open(nugetConfigPath)
 	panicErrorIfAny(err, "Open " + nugetConfigPath)
 	parsed, err := xmlquery.Parse(f)
@@ -66,49 +101,71 @@ func initNugetIndexJsonCache(pkgName string) {
 
 		auth_entries := xmlquery.Find(parsed, "//configuration/packageSourceCredentials/" + sname + "/add")
 		for _, auth_entry := range auth_entries {
+			outputVarForThisEntry := &sauth_username
 			for _, attr := range auth_entry.Attr {
-				if attr.Name.Local == "Username" {
-					sauth_username = attr.Value
-				} else if attr.Name.Local == "ClearTextPassword" {
-					sauth_password = attr.Value
+				if attr.Name.Local == "key"  && attr.Value == "ClearTextPassword"{
+					outputVarForThisEntry = &sauth_password
+				} else if attr.Name.Local == "value" {
+					*outputVarForThisEntry = attr.Value
 				}
 			}
 		}
 
 		// Send the HTTP request and download index.json!
 		wg.Add(1)
-		nugetIndexJsonCache = append(nugetIndexJsonCache, "")
 		go func(url, authu, authp string) {
 			defer wg.Done()
-			req, err := http.NewRequest("GET", url, nil)
-			panicErrorIfAny(err, "http.NewRequest")
-			if authu + authp != "" {
-				req.SetBasicAuth(authu, authp)
+
+			// This message could be cached, to speedup.
+			source_desc_json := httpGet(url, authu, authp, "")
+			if len(source_desc_json) == 0 {
+				return
 			}
-			resp, err := http.DefaultClient.Do(req)
-			logErrorIfAny(err, "download json")
-			defer resp.Body.Close()
-			resp_txt, err := ioutil.ReadAll(resp.Body)
-			logErrorIfAny(err, "download json")
 
-			if len(resp_txt) > 0 {
+			// The golang json parse library sucks. Prevent it from crashing
+			//   on a json syntax error.
+			defer func() {recover()} ()
+
+			var parsedJson map[string]interface{}
+			json.Unmarshal([]byte(source_desc_json), &parsedJson)
+			endpoints := parsedJson["resources"].([]interface{})
+			selectedEndpoint := ""
+			for _, endpoint_ := range endpoints {
+				endpoint := endpoint_.(map[string]interface{})
+				if endpoint["@type"] == "RegistrationsBaseUrl/Versioned" {
+					selectedEndpoint = endpoint["@id"].(string)
+				}
+			}
+
+			respTxt := httpGet(selectedEndpoint + pkgName + "/index.json", authu, authp, "")
+			if len(respTxt) > 0 && ! strings.Contains(respTxt, "Can't find the package ") {
 				nugetIndexJsonCache_wlock.Lock()
-				nugetIndexJsonCache = append(nugetIndexJsonCache, string(resp_txt))
+				nugetIndexJsonCache = append(nugetIndexJsonCache, respTxt)
+				nugetIndexJsonCache_Login = append(nugetIndexJsonCache_Login, authu + ":" + authp)
 				nugetIndexJsonCache_wlock.Unlock()
 			}
 		}(surl, sauth_username, sauth_password)
 	}
 	wg.Wait()
+	return
 }
 
 // recursive. If package exists, it exit.
 // One of [pkgVerBegin, pkgVerEnd] would be downloaded.
-func downloadPackageAndDeps(packageName, pkgVerBegin, pkgVerEnd, localRepoDir string) {
+func downloadPackageAndDeps(packageName, pkgVerBegin, pkgVerEnd, localRepoDir string, indexJsons []string) {
+
+}
+
+// non-recursive. If package exists, it exit.
+// One of [pkgVerBegin, pkgVerEnd] would be downloaded.
+func downloadPackage(packageName, pkgVer, localRepoDir string, indexJsons []string) {
 
 }
 
+
 // [pkgVerBegin, pkgVerEnd] is good for frameworkVersion.
-func decidePackageVersion(packageName, frameworkVersion string) (pkgVerBegin, pkgVerEnd string) {
+// This function was designed to download dependencies.
+func decidePackageVersionRange(packageName, frameworkVersion string, indexJsons []string) (pkgVerBegin, pkgVerEnd string) {
 	// Generated by curl-to-Go: https://mholt.github.io/curl-to-go
 	// curl -u bensl:xbwejuparq4ighqs4pjq6bqjqmxpzghhtqshql2uy3woi73ew6iq https://o365exchange.pkgs.visualstudio.com/959adb23-f323-4d52-8203-ff34e5cbeefa/_packaging/aacaa93f-61ac-420c-8b20-c17c5d2b52f1/nuget/v3/registrations2-semver2/microsoft.azure.kusto.cloud.platform/index.json
 
@@ -116,6 +173,71 @@ func decidePackageVersion(packageName, frameworkVersion string) (pkgVerBegin, pk
 	return "1.1.1", "8.8.8"
 }
 
+// It's too hard to parse the json... Let's get everything we need in one time.
+// If frameworkVersion != "", then I find the latest available package for this framework.
+// If packageVersion != "", then I find the url for this version.
+// You must only set ONE OF THEM! If you set both, the `frameworkVersion` would be ignored.
+func decidePackageVersion(frameworkVersion, packageVersion string, indexJsons, indexJsons_Login []string) (maxAvailableVersion, maxAvailableVersionUrl, maxAvailableVersionUrlLogin string) {
+	// The golang json parse library sucks. Prevent it from crashing
+	//   on a json syntax error.
+	defer func() {recover()} ()
+
+	for indexJson_currIndex, indexJson := range indexJsons {
+		defer func() {recover()} () // try parse, skip this entry on error
+		// entry = xml.findall("/items/[]/items/[]/catalogEntry")
+		// entry/version, entry/dependencyGroups[]/targetFramework, entry/../packageContent
+		// If there's no `targetFramework` in dependencyGroups, this package depends on specific version of parent package. it sucks.
+		var j1 map[string]interface{}
+		json.Unmarshal([]byte(indexJson), &j1)
+		j2 := j1["items"].([]interface{})
+		for _, j2ele := range j2 {
+			defer func() {recover()} () // try parse, skip this entry on error
+			j3 := j2ele.(map[string]interface{})["items"]
+			j4 := j3.([]interface{})
+			for _, jPkgVersionItem_ := range j4 {
+				defer func() {recover()} () // try parse, skip this entry on error
+				jPkgVersionItem := jPkgVersionItem_.(map[string]interface{})
+				jCatalogEntry := jPkgVersionItem["catalogEntry"].(map[string]interface{})
+				j5 := jCatalogEntry["dependencyGroups"].([]interface{})
+				myVersionText := jCatalogEntry["version"].(string)
+
+				// Let's check if this version is ok
+				thisPkgVersionIsOk := false
+
+				if frameworkVersion != "" {
+					for _, j6 := range j5 {
+						j7 := j6.(map[string]interface{})
+						if j8, ok := j7["targetFramework"]; ok {
+							thisPkgVersionIsOk = frameworkVersion == netVersionNugetFormatToStandardFormat(j8.(string))
+							break
+						} else {
+							// if not ok, then this pkgVersion has no targetFramework limitation.
+							thisPkgVersionIsOk = true
+						}
+					}
+				}
+
+				if packageVersion != "" {
+					thisPkgVersionIsOk = packageVersion == myVersionText
+				}
+
+				if thisPkgVersionIsOk {
+					// This version is ok! Log it.
+					myVersionUrl := jPkgVersionItem["packageContent"].(string)
+					if version.CompareSimple(myVersionText, maxAvailableVersion) == 1 {
+						// if left > right
+						maxAvailableVersion = myVersionText
+						maxAvailableVersionUrl = myVersionUrl
+						maxAvailableVersionUrlLogin = indexJsons_Login[indexJson_currIndex]
+					}
+				}
+			}
+		}
+	}
+
+	return
+}
+
 func main() {
 	if(len(os.Args) < 4) {
 		log.Println("Usage: nuget-download-package <packageName> <packageVersion/frameworkVersion> <localRepoDir>")
@@ -123,11 +245,31 @@ func main() {
 		os.Exit(1)
 	}
 
-    pkgName, pkgVer, localRepoDir := os.Args[1], os.Args[2], os.Args[3]
-    if strings.HasPrefix(pkgVer, "net") {
-    	_, pkgVer = decidePackageVersion(pkgName, pkgVer)
+    pkgName, pkgVerOrNetVer, localRepoDir := strings.ToLower(os.Args[1]), os.Args[2], os.Args[3]
+    indexJsons, indexJsons_Login := initNugetIndexJsonCache(pkgName)
+
+	pkgVer, targetUrl, targetUrlLogin := "", "", ""
+    if strings.HasPrefix(pkgVerOrNetVer, "net") {
+    	pkgVer, targetUrl, targetUrlLogin = decidePackageVersion(pkgVerOrNetVer, "", indexJsons, indexJsons_Login)
+	} else {
+		pkgVer, targetUrl, targetUrlLogin = decidePackageVersion("", pkgVerOrNetVer, indexJsons, indexJsons_Login)
+	}
+
+	log.Println("Using pkgVer " + pkgVer + " from " + targetUrl)
+
+	pkgBaseDir := filepath.Join(localRepoDir, pkgName, pkgVer)
+	err := os.RemoveAll(pkgBaseDir)
+	logErrorIfAny(err, "rm -rf " + pkgBaseDir)
+	err = os.MkdirAll(pkgBaseDir, 0777)
+	logErrorIfAny(err, "mkdir -p " + pkgBaseDir)
+	if targetUrlLogin == "" {
+		httpGet(targetUrl, "", "", filepath.Join(pkgBaseDir, "openxt.pkgdown.zip"))
+	} else {
+		s := strings.Split(targetUrlLogin, ":")
+		httpGet(targetUrl, s[0], s[1], filepath.Join(pkgBaseDir, "openxt.pkgdown.zip"))
 	}
 
-	downloadPackageAndDeps(pkgName, pkgVer, pkgVer, localRepoDir)
+	err = Extract(filepath.Join(pkgBaseDir, "openxt.pkgdown.zip"), pkgBaseDir)
+	logErrorIfAny(err, "unzip openxt.pkgdown.zip")
 }
 

tools/nuget-download-package/unzip.go

+package main
+
+import (
+	"archive/zip"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+
+func Extract(src, dest string) error {
+	fmt.Println("Extraction of " + src + " started!")
+
+	r, err := zip.OpenReader(src)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := r.Close(); err != nil {
+			panic(err)
+		}
+	}()
+
+	os.MkdirAll(dest, 0777)
+
+	// Closure to address file descriptors issue with all the deferred .Close() methods
+	extractAndWriteFile := func(f *zip.File) error {
+		rc, err := f.Open()
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if err := rc.Close(); err != nil {
+				panic(err)
+			}
+		}()
+
+		path := filepath.Join(dest, f.Name)
+		if !strings.HasPrefix(path, filepath.Clean(dest)+string(os.PathSeparator)) {
+			return fmt.Errorf("%s: Illegal file path", path)
+		}
+
+		if f.FileInfo().IsDir() {
+			os.MkdirAll(path, 0777) // f.Mode())
+		} else {
+			os.MkdirAll(filepath.Dir(path), 0777) // f.Mode())
+			f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				if err := f.Close(); err != nil {
+					panic(err)
+				}
+			}()
+
+			_, err = io.Copy(f, rc)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+
+	for _, f := range r.File {
+		err := extractAndWriteFile(f)
+		if err != nil {
+			return err
+		}
+		fmt.Println("Extracting file: " + f.Name)
+	}
+
+	fmt.Println("Extraction of " + src + " finished!")
+
+	return nil
+}