Merge branch 'main' of https://git.wamblee.org/public/gotools

cmd/go2junit/go2junit.go

@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 	"runtime/debug"
 	"strings"
+	"text/tabwriter"
 )
 
 func getVersion() string {
@@ -140,19 +141,24 @@ func main() {
 	}
 
 	fmt.Printf("\n\nSUMMARY\n\n")
-	fmt.Printf("%-60s %-10s %-10s %-10s %-10s %-10s %-10s\n\n", "SUITE", "COUNT", "PASSED", "FAILURES", "ERRORS", "DISABLED", "SKIPPED")
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
+
+	fmt.Fprintf(w, "%s\t%-10s\t%-10s\t%-10s\t%-10s\t%-10s\t%-10s\n",
+		"SUITE", "COUNT", "PASSED", "FAILURES", "ERRORS", "DISABLED", "SKIPPED")
+
 	for _, suite := range testsuites.Suites {
-		fmt.Printf("%-60s %-10d %-10d %-10d %-10d %-10d %-10d\n",
+		fmt.Fprintf(w, "%s\t%-10d\t%-10d\t%-10d\t%-10d\t%-10d\t%-10d\n",
 			suite.Name,
 			suite.TestCount,
 			suite.TestCount-suite.Failures-suite.Errors-suite.Skipped-suite.Disabled,
 			suite.Failures, suite.Errors, suite.Disabled, suite.Skipped)
 	}
-	fmt.Printf("\n%-60s %-10d %-10d %-10d %-10d %-10d %-10d\n",
+	fmt.Fprintf(w, "%s\t%-10d\t%-10d\t%-10d\t%-10d\t%-10d\t%-10d\n",
 		"TOTAL",
 		testsuites.Tests,
 		testsuites.Tests-testsuites.Failures-testsuites.Errors-testsuites.Skipped-testsuites.Disabled,
 		testsuites.Failures, testsuites.Errors, testsuites.Disabled, testsuites.Skipped)
+	w.Flush()
 
 	if testsuites.Failures+testsuites.Errors+testsuites.Skipped+testsuites.Disabled > 0 {
 		fmt.Printf("\nFAILED TESTS\n\n")

cmd/lcstest/main.go

@@ -0,0 +1,137 @@
+package main
+
+import (
+	"fmt"
+	"math/rand"
+	"time"
+)
+
+// from: https://github.com/peter-evans/patience/blob/main/lcs.go
+func LCS[T comparable](a, b []T, equals func(T, T) bool) [][2]int {
+	// Initialize the LCS table.
+	lcs := make([][]int, len(a)+1)
+	for i := 0; i <= len(a); i++ {
+		lcs[i] = make([]int, len(b)+1)
+	}
+
+	// Populate the LCS table.
+	for i := 1; i < len(lcs); i++ {
+		for j := 1; j < len(lcs[i]); j++ {
+			if equals(a[i-1], b[j-1]) {
+				lcs[i][j] = lcs[i-1][j-1] + 1
+			} else {
+				lcs[i][j] = max(lcs[i-1][j], lcs[i][j-1])
+			}
+		}
+	}
+
+	// Backtrack to find the LCS.
+	i, j := len(a), len(b)
+	s := make([][2]int, 0, lcs[i][j])
+
+	for i > 0 && j > 0 {
+		switch {
+		case equals(a[i-1], b[j-1]):
+			s = append(s, [2]int{i - 1, j - 1})
+			i--
+			j--
+		case lcs[i-1][j] > lcs[i][j-1]:
+			i--
+		default:
+			j--
+		}
+	}
+
+	// Reverse the backtracked LCS.
+	for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
+		s[i], s[j] = s[j], s[i]
+	}
+
+	return s
+}
+
+func LCSDiff[T comparable](a, b []T, equals func(T, T) bool,
+	diff func(T), same func(T, T)) {
+	res := LCS(a, b, equals)
+
+	i2 := 0
+	isame := 0
+	for isame < len(res) {
+		// process [i1..isamee1[ and [i2..isame2[
+		for i2 < res[isame][1] {
+			diff(b[i2])
+			i2++
+		}
+		// process same elements -> no diff so normally not
+		same(a[res[isame][0]], b[res[isame][1]])
+		i2++
+		isame++
+	}
+	for i2 < len(b) {
+		diff(b[i2])
+		i2++
+	}
+}
+
+const letters = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+
+func randString(n int) string {
+	res := make([]byte, n)
+	for i := range n {
+		res[i] = letters[rand.Int()%len(letters)]
+	}
+	return string(res)
+}
+
+func main() {
+	input1 := []string{"ab", "cd", "myidenticalstring"}
+	input2 := []string{"ab", "ce", "myidenticalstrings", "fg"}
+	t0 := time.Now()
+	res := LCS(input1, input2, func(s1 string, s2 string) bool {
+		res := LCS([]rune(s1), []rune(s2), func(r rune, r2 rune) bool {
+			return r == r2
+		})
+		score := min(float64(len(res))/float64(len(s1)),
+			float64(len(res))/float64(len(s2)))
+		return score > 0.90
+	})
+	dt := time.Now().Sub(t0).Microseconds()
+	fmt.Printf("time %v us, len %v\n", dt, len(res))
+	for i, pair := range res {
+		fmt.Printf("%d: %v\n", i, pair)
+	}
+
+	// [0..i1[ and [0..i2[ already handled
+	// isame: res[0..isame[ have been handled
+	//i1 := 0
+	i2 := 0
+	isame := 0
+	for isame < len(res) {
+		// process [i1..isamee1[ and [i2..isame2[
+		for i2 < res[isame][1] {
+			fmt.Printf("DIFF|ADD %s\n", input2[i2])
+			i2++
+		}
+		// process same elements -> no diff so normally not
+		fmt.Printf("SAME     %s %s\n", input1[res[isame][0]], input2[res[isame][1]])
+		i2++
+		isame++
+	}
+	for i2 < len(input2) {
+		fmt.Printf("DIFF|ADD %s\n", input2[i2])
+		i2++
+	}
+
+	LCSDiff(input1, input2, func(s1 string, s2 string) bool {
+		res := LCS([]rune(s1), []rune(s2), func(r rune, r2 rune) bool {
+			return r == r2
+		})
+		score := min(float64(len(res))/float64(len(s1)),
+			float64(len(res))/float64(len(s2)))
+		return score > 0.90
+	}, func(s string) {
+		fmt.Printf("DIFF %v\n", s)
+	}, func(s1 string, s2 string) {
+		fmt.Printf("SAME %v %v\n", s1, s2)
+	})
+}

cmd/yamltool/diff.go

@@ -1,54 +1,14 @@
 package main
 
 import (
-	"bytes"
 	"fmt"
-	yaml "github.com/goccy/go-yaml"
-	"github.com/spf13/cobra"
 	"os"
 	"reflect"
+
+	"github.com/goccy/go-yaml"
+	"github.com/spf13/cobra"
 )
 
-var VERBOSITY = 2
-
-func read(file string) []byte {
-	data, err := os.ReadFile(file)
-	if err != nil {
-		panic(err)
-	}
-	return data
-}
-
-func parse(data []byte) yaml.MapSlice {
-	var result yaml.MapSlice
-	decoder := yaml.NewDecoder(bytes.NewReader(data),
-		yaml.UseOrderedMap())
-	err := decoder.Decode(&result)
-	if err != nil {
-		panic(err)
-	}
-	return result
-}
-
-type TypeId int
-
-const (
-	Map TypeId = iota
-	Slice
-	Scalar
-)
-
-func Type(elem any) TypeId {
-	switch elem.(type) {
-	case yaml.MapSlice:
-		return Map
-	case []any:
-		return Slice
-	default:
-		return Scalar
-	}
-}
-
 // hack to be able to compare slices and dictionires that cannot be put into a map.
 func strval(v any) string {
 	return fmt.Sprintf("%v", v)
@@ -86,7 +46,13 @@ func subtract(yaml2 yaml.MapSlice, yaml1 yaml.MapSlice) yaml.MapSlice {
 				v1set[strval(v)] = true
 			}
 			s := make([]any, 0)
-			for k2, _ := range v2set {
+			// TODO
+			//   convert both slices to lists of strings
+			//   apply LCS to the list of strings with approximate equality
+			//   added elements:  -> output fully
+			//   approximately equal elements: -> when identical, skip, otherwise, output diffs (recurse)
+			for _, v2value := range v2.([]any) {
+				k2 := strval(v2value)
 				if v1set[k2] {
 					if VERBOSITY == 2 {
 						s = append(s, "<UNMODIFIED>")
@@ -108,44 +74,56 @@ func subtract(yaml2 yaml.MapSlice, yaml1 yaml.MapSlice) yaml.MapSlice {
 	return res
 }
 
-func execute(cmd *cobra.Command, args []string) error {
+func diff(cmd *cobra.Command, args []string) error {
 	if len(args) != 2 {
-		return fmt.Errorf("Parameters expected")
+		return fmt.Errorf("Expected 2 files")
 	}
-	if VERBOSITY < 1 || VERBOSITY > 3 {
+	if VERBOSITY < 0 || VERBOSITY > 3 {
 		return fmt.Errorf("Array verbosity out of range")
 	}
-	file1 := os.Args[1]
-	file2 := os.Args[2]
+	file1 := args[0]
+	file2 := args[1]
 
-	yaml1 := parse(read(file1))
-	yaml2 := parse(read(file2))
-
-	yaml2 = subtract(yaml2, yaml1)
-
-	enc := yaml.NewEncoder(os.Stdout,
-		yaml.UseLiteralStyleIfMultiline(true),
-		yaml.Indent(2), // Set indentation
-		//yaml.UseOrderedMap(), // Preserve map order
-	)
-	err := enc.Encode(yaml2)
-	return err
-}
-
-func main() {
-
-	cmd := &cobra.Command{
-		Use:   "yamldiff <file1> <file2>",
-		Short: "Shows one-way difference between yaml files",
-		Long: `
-Shows the changes in <file2> compared to <file1>`,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			return execute(cmd, args)
-		},
+	data1, err := read(file1)
+	if err != nil {
+		return err
+	}
+	data2, err := read(file2)
+	if err != nil {
+		return err
 	}
 
-	cmd.PersistentFlags().IntVarP(&VERBOSITY, "array-output-level",
-		"v", 1, "Array output level: , 1: only show changed/added values, 2 show identical as <UNMODIFIED>, 3: show all values")
+	yaml1, err := parse(data1)
+	if err != nil {
+		return fmt.Errorf("%s: %w", file1, err)
+	}
+	yaml2, err := parse(data2)
+	if err != nil {
+		return fmt.Errorf("%s: %w", file2, err)
+	}
 
-	cmd.Execute()
+	diff1 := subtract(yaml2, yaml1)
+	diff2 := make(yaml.MapSlice, 0)
+	if SYMMETRIC_DIFF {
+		diff2 = subtract(yaml1, yaml2)
+	}
+
+	diff := diff1
+	if SYMMETRIC_DIFF {
+		diff = make(yaml.MapSlice, 0)
+		diff = append(diff,
+			yaml.MapItem{Key: "forward", Value: diff1},
+			yaml.MapItem{Key: "backward", Value: diff2},
+		)
+	}
+
+	if VERBOSITY > 0 {
+		if err := encode(os.Stdout, diff); err != nil {
+			return err
+		}
+	}
+	if len(diff1) > 0 || len(diff2) > 0 {
+		os.Exit(1)
+	}
+	return nil
 }

cmd/yamltool/encode.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+	"github.com/goccy/go-yaml"
+	"io"
+	"os"
+)
+
+func encode(writer io.Writer, data any) error {
+	enc := yaml.NewEncoder(os.Stdout,
+		yaml.UseLiteralStyleIfMultiline(true),
+		yaml.Indent(2), // Set indentation
+		//yaml.UseOrderedMap(), // Preserve map order
+	)
+	return enc.Encode(data)
+}

cmd/yamltool/merge.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+
+	"github.com/goccy/go-yaml"
+	"github.com/spf13/cobra"
+)
+
+type MyMap yaml.MapSlice
+
+func (m *MyMap) Set(key any, value any) {
+	for i := range len(*m) {
+		if (*m)[i].Key == key {
+			(*m)[i].Value = value
+			return
+		}
+	}
+	*m = append(*m, yaml.MapItem{Key: key, Value: value})
+}
+
+func (m MyMap) Get(key any) any {
+	for _, item := range m {
+		if item.Key == key {
+			return item.Value
+		}
+	}
+	return nil
+}
+
+func mergeMap(yaml1 yaml.MapSlice, yaml2 yaml.MapSlice) yaml.MapSlice {
+	res := MyMap(yaml1)
+
+	for _, item := range yaml2 {
+		initialValue := res.Get(item.Key)
+		value := item.Value
+		switch {
+		case initialValue != nil:
+			if reflect.TypeOf(initialValue) == reflect.TypeOf(yaml.MapSlice{}) &&
+				reflect.TypeOf(value) == reflect.TypeOf(yaml.MapSlice{}) {
+				mergedMap := mergeMap(initialValue.(yaml.MapSlice), value.(yaml.MapSlice))
+				res.Set(item.Key, mergedMap)
+			} else {
+				res.Set(item.Key, item.Value)
+			}
+		default:
+			res.Set(item.Key, item.Value)
+		}
+	}
+	return yaml.MapSlice(res)
+}
+
+func merge(cmd *cobra.Command, args []string) error {
+	res := make(yaml.MapSlice, 0)
+	for _, arg := range args {
+		data, err := read(arg)
+		if err != nil {
+			return err
+		}
+		config, err := parse(data)
+		if err != nil {
+			return fmt.Errorf("%s: %w", arg, err)
+		}
+		res = mergeMap(res, config)
+	}
+	encode(os.Stdout, res)
+	return nil
+}

cmd/yamltool/parse.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/goccy/go-yaml"
+	"github.com/spf13/cobra"
+)
+
+func parse(data []byte) (yaml.MapSlice, error) {
+	var result yaml.MapSlice
+	decoder := yaml.NewDecoder(bytes.NewReader(data),
+		yaml.UseOrderedMap())
+	err := decoder.Decode(&result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func parseFiles(cmd *cobra.Command, args []string) error {
+	for _, arg := range args {
+		data, err := read(arg)
+		if err != nil {
+			return err
+		}
+		_, err = parse(data)
+		if err != nil {
+			fmt.Printf("%s: %v\n", arg, err.Error())
+		}
+	}
+	return nil
+}

cmd/yamltool/sort.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"cmp"
+	"fmt"
+	"reflect"
+	"slices"
+
+	"github.com/goccy/go-yaml"
+	"github.com/spf13/cobra"
+
+	"os"
+)
+
+type MapSlice yaml.MapSlice
+
+func (s MapSlice) Sort() {
+	slices.SortFunc(s, func(a, b yaml.MapItem) int {
+		keya := fmt.Sprintf("%s", a.Key)
+		keyb := fmt.Sprintf("%s", b.Key)
+		return cmp.Compare(keya, keyb)
+	})
+	for _, item := range s {
+		switch {
+		case reflect.TypeOf(item.Value) == reflect.TypeOf(yaml.MapSlice{}):
+			((MapSlice)(item.Value.(yaml.MapSlice))).Sort()
+		case Type(item.Value) == Slice:
+			for _, v := range item.Value.([]any) {
+				ms, ok := v.(yaml.MapSlice)
+				if ok {
+					((MapSlice)(ms)).Sort()
+				}
+			}
+		}
+	}
+}
+
+func sortYaml(cmd *cobra.Command, args []string) error {
+	for _, arg := range args {
+		data, err := read(arg)
+		if err != nil {
+			return err
+		}
+		doc, err := parse(data)
+		if err != nil {
+			return fmt.Errorf("%s: %v\n", arg, err.Error())
+		}
+		((MapSlice)(doc)).Sort()
+		if len(args) > 1 {
+			fmt.Printf("---\n")
+		}
+		encode(os.Stdout, doc)
+	}
+	return nil
+}

cmd/yamltool/support.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+)
+
+func read(file string) ([]byte, error) {
+
+	if file == "-" {
+		data, err := io.ReadAll(os.Stdin)
+		if err != nil {
+			return nil, fmt.Errorf("Error reading from stdin")
+		}
+		return data, nil
+	}
+	data, err := os.ReadFile(file)
+	if err != nil {
+		return nil, fmt.Errorf("Error reading from '%s'", file)
+	}
+	return data, nil
+}

cmd/yamltool/yamltool.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	yaml "github.com/goccy/go-yaml"
+	"github.com/spf13/cobra"
+)
+
+var VERBOSITY = 3
+var SYMMETRIC_DIFF = false
+
+type TypeId int
+
+const (
+	Map TypeId = iota
+	Slice
+	Scalar
+)
+
+func Type(elem any) TypeId {
+	switch elem.(type) {
+	case yaml.MapSlice:
+		return Map
+	case []any:
+		return Slice
+	default:
+		return Scalar
+	}
+}
+
+func main() {
+
+	cmd := &cobra.Command{
+		Use:   "yamltool",
+		Short: "Shows one-way difference between yaml files",
+		Long: `
+Shows the changes in <file2> compared to <file1>`,
+	}
+
+	diff := &cobra.Command{
+		Use:   "diff [file1] [file2]",
+		Short: "Shows one-way difference between yaml files",
+		Long: `
+Shows the additions and modifications in <file2> compared to <file1>`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return diff(cmd, args)
+		},
+	}
+	cmd.AddCommand(diff)
+
+	merge := &cobra.Command{
+		Use:   "merge [file1] ... [fileN]",
+		Short: "Merge yaml files.",
+		Long:  `Changes will be merged into the first file, so later files override earlier ones`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return merge(cmd, args)
+		},
+	}
+	cmd.AddCommand(merge)
+
+	parse := &cobra.Command{
+		Use:   "parse [file1] ... [fileN]",
+		Short: "Parse yaml files.",
+		Long:  `Parse yaml files, usually gives better error messages than yamllint or yq`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return parseFiles(cmd, args)
+		},
+	}
+	cmd.AddCommand(parse)
+
+	sort := &cobra.Command{
+		Use:   "sort [file1] ... [fileN]",
+		Short: "Sort the yaml output by sorting based on map key ",
+		Long:  `Sort yaml files, this makes it easier to also use regular diff`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return sortYaml(cmd, args)
+		},
+	}
+	cmd.AddCommand(sort)
+
+	diff.PersistentFlags().IntVarP(&VERBOSITY, "array-output-level",
+		"v", 3, `Array output level: ,
+	0: no output, only exit status,
+	1: only show changed/added values, 
+	2: show identical as <UNMODIFIED>, 
+	3: show all values`)
+	diff.Flags().BoolVarP(&SYMMETRIC_DIFF, "symmetric-diff",
+		"s", false, `Symmetric difference, compare in both directions`)
+
+	cmd.Execute()
+}

go.mod

@@ -1,6 +1,6 @@
 module git.wamblee.org/public/gotools
 
-go 1.23.3
+go 1.24.7
 
 require (
 	github.com/goccy/go-yaml v1.15.13