examples_test.go

Documentation: github.com/klauspost/compress/s2

     1  // Copyright (c) 2023+ Klaus Post. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package s2_test
     6  
     7  import (
     8  	"bytes"
     9  	"fmt"
    10  	"os"
    11  
    12  	"github.com/klauspost/compress/s2"
    13  	"github.com/klauspost/compress/zstd"
    14  )
    15  
    16  func ExampleMakeDict() {
    17  	// Read a sample
    18  	sample, err := os.ReadFile("../testdata/gettysburg.txt")
    19  	if err != nil {
    20  		panic(err)
    21  	}
    22  	fmt.Println("Input size:", len(sample))
    23  
    24  	// Create a dictionary.
    25  	dict := s2.MakeDict(sample, nil)
    26  	fmt.Println("Dict size:", len(dict.Bytes()))
    27  
    28  	encoded := dict.Encode(nil, sample)
    29  	if len(encoded) < 20 {
    30  		fmt.Println("Encoded size was less than 20 bytes!")
    31  	}
    32  
    33  	// To decode:
    34  	decoded, err := dict.Decode(nil, encoded)
    35  	if err != nil {
    36  		panic(err)
    37  	}
    38  	if bytes.Equal(decoded, sample) {
    39  		fmt.Println("They match!")
    40  	}
    41  	// OUTPUT:
    42  	// Input size: 1548
    43  	// Dict size: 1549
    44  	// Encoded size was less than 20 bytes!
    45  	// They match!
    46  }
    47  
    48  func ExampleMakeDict_zstd() {
    49  	// Read dictionary generated by zStandard using the command line
    50  	// λ zstd -r --train-fastcover -o zstd.dict --maxdict=2048 gosrc\*
    51  	// With gosrc containing all the standard library source files.
    52  	zdict := []byte("7\xa40콶\xc1\x1bB\x10\x982\xc4\xe9\xc0\xc0\xc0\xc0\xc0\xc0\xc0\xc0\xc0\xc0\xc0\xc0@\xf5<\xda#\"{\xb7\xb6\xdd\xdd\xda\x17\x1b\t\x9b\xbd\x13n{U\xc1k\x11\xc3\x1b\x8b\xfbX\xee\xfe\xcb1\xcai\f\xf6meE\x97\x19\x83\\f\x14\x00\\\tS\x01\x00\x18 \x18\x8f\aT\x1a\xf5\x00\x00\x04\x80O\xd3MIJH\x03q\x98$I\n\xa3\x10B\xc6\x18B\b\x01\x00\x00D\x00\x04\x04\x00\xc0\x00\x00\x004\xcdieĩ@Β \xc7\x14B\n͌\b\x00\x00\x00\x00\x01\x00\x00\x00\x04\x00\x00\x00\b\x00\x00\x00kage types2\n\nimport (\n\t\"cmd/compile/internal/syntax\"\n\t\"strings\"\n\t\"unicode\"\n)\n\n// funcInst type-checks a func\")\n\tif err != nil {\n\t\tt.Fatalf(\"Prepare: %v\", err)\n\t}\n\tdefer stmt.Close()\n\n\tconst n = 10\n\tch := make(chan error, n)\n\tfor i := 0; i < n; i++ {\n\t\tgo func() {\n\t\t\tvar age int\n\t\t\terr := stmt.QueryRowool { return c != nil && c.fd != nil }\n\n// Implementation of the Conn interface.\n\n// Read implements the Conn Read method.\nfunc (c *conn) Read(b []byte) (int, error) {\n\tif !c.ok() {\n\t\treturn 0, t\n\t\t} else {\n\t\t\treturn nil, &FormatError{0, \"invalid magic number\", nil}\n\t\t}\n\t}\n\toffset := int64(4)\n\n\t// Read the number of FatArchHeaders that come after the fat_header.\n\tvar narch uint32\n\terr log.Fatal(err)\n\t\t}\n\t\tf := strings.Fields(line)\n\t\tif len(f) == 0 {\n\t\t\tcontinue\n\t\t}\n\t\tswitch f[0] {\n\t\tdefault:\n\t\t\tfmt.Fprintf(os.Stderr, \"?unknown command\\n\")\n\t\t\tcontinue\n\t\tcase \"tags\":\n\t\t\tprefix 00\\x00\\x00\", true},\n\t}\n\n\tfor _, v := range vectors {\n\t\tvar f formatter\n\t\tgot := make([]byte, len(v.want))\n\t\tf.formatNumeric(got, v.in)\n\t\tok := (f.err == nil)\n\t\tif ok != v.ok {\n\t\t\tif v.ok {\n\t\t\t\ttturn true\n\t}\n\treturn false\n}\nfunc rewriteValueARM_OpARMBICconst(v *Value) bool {\n\tv_0 := v.Args[0]\n\t// match: (BICconst [0] x)\n\t// result: x\n\tfor {\n\t\tif auxIntToInt32(v.AuxInt) != 0 {\n\t\t\tbreak\n\tnt) {\n\t\t\t\t\tt.Errorf(\"%5g %s %5g = %5s; want %5s\", x, op, y, got, want)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestFloatArithmeticOverflow(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tprec       uint\n\t\t)\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n}\n// Copyright 2017 The Go Authors. All rights reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\npackage ), uintptr(unsafe.Pointer(_p1)), 0)\n\tif e1 != 0 {\n\t\terr = errnoErr(e1)\n\t}\n\treturn\n}\n\n// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT\n\nfunc Sync() (err error) {\n\t_, _, e1 := SyscDLINK       = 0x10\n\tMOVEFILE_FAIL_IF_NOT")
    53  
    54  	// Decode the zstandard dictionary.
    55  	insp, err := zstd.InspectDictionary(zdict)
    56  	if err != nil {
    57  		panic(err)
    58  	}
    59  
    60  	// We are only interested in the contents.
    61  	fmt.Println("Dictionary content length:", len(insp.Content()))
    62  
    63  	// Create a dictionary.
    64  	// Assume that files start with "// Copyright (c) 2023".
    65  	// Search for the longest match for that.
    66  	// This may save a few bytes.
    67  	dict := s2.MakeDict(insp.Content(), []byte("// Copyright (c) 2023"))
    68  
    69  	// b := d.Bytes() will provide a dictionary that can be saved
    70  	// and reloaded with s2.NewDict(b).
    71  
    72  	fmt.Println("Dict size:", len(dict.Bytes()))
    73  
    74  	// Read a sample. Use this file.
    75  	sample, err := os.ReadFile("examples_test.go")
    76  	if err != nil {
    77  		panic(err)
    78  	}
    79  
    80  	encodedWithDict := dict.Encode(nil, sample)
    81  	encodedNoDict := s2.Encode(nil, sample)
    82  
    83  	// Print a less accurate output that is less likely to change.
    84  	// Since we include the (encoded) dictionary itself that will create better than expected compression.
    85  	if len(encodedWithDict) < len(encodedNoDict)-1000 {
    86  		fmt.Println("Saved more than 1000 bytes")
    87  	}
    88  
    89  	// To decode the content:
    90  	decoded, err := dict.Decode(nil, encodedWithDict)
    91  	if err != nil {
    92  		panic(err)
    93  	}
    94  	if bytes.Equal(decoded, sample) {
    95  		fmt.Println("They match!")
    96  	}
    97  	// OUTPUT:
    98  	// Dictionary content length: 1894
    99  	// Dict size: 1896
   100  	// Saved more than 1000 bytes
   101  	// They match!
   102  }
   103
View as plain text