...
Run Format

Source file src/compress/flate/huffman_code.go

     1	// Copyright 2009 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	package flate
     6	
     7	import (
     8		"math"
     9		"sort"
    10	)
    11	
    12	// hcode is a huffman code with a bit code and bit length.
    13	type hcode struct {
    14		code, len uint16
    15	}
    16	
    17	type huffmanEncoder struct {
    18		codes     []hcode
    19		freqcache []literalNode
    20		bitCount  [17]int32
    21		lns       byLiteral // stored to avoid repeated allocation in generate
    22		lfs       byFreq    // stored to avoid repeated allocation in generate
    23	}
    24	
    25	type literalNode struct {
    26		literal uint16
    27		freq    int32
    28	}
    29	
    30	// A levelInfo describes the state of the constructed tree for a given depth.
    31	type levelInfo struct {
    32		// Our level.  for better printing
    33		level int32
    34	
    35		// The frequency of the last node at this level
    36		lastFreq int32
    37	
    38		// The frequency of the next character to add to this level
    39		nextCharFreq int32
    40	
    41		// The frequency of the next pair (from level below) to add to this level.
    42		// Only valid if the "needed" value of the next lower level is 0.
    43		nextPairFreq int32
    44	
    45		// The number of chains remaining to generate for this level before moving
    46		// up to the next level
    47		needed int32
    48	}
    49	
    50	// set sets the code and length of an hcode.
    51	func (h *hcode) set(code uint16, length uint16) {
    52		h.len = length
    53		h.code = code
    54	}
    55	
    56	func maxNode() literalNode { return literalNode{math.MaxUint16, math.MaxInt32} }
    57	
    58	func newHuffmanEncoder(size int) *huffmanEncoder {
    59		return &huffmanEncoder{codes: make([]hcode, size)}
    60	}
    61	
    62	// Generates a HuffmanCode corresponding to the fixed literal table
    63	func generateFixedLiteralEncoding() *huffmanEncoder {
    64		h := newHuffmanEncoder(maxNumLit)
    65		codes := h.codes
    66		var ch uint16
    67		for ch = 0; ch < maxNumLit; ch++ {
    68			var bits uint16
    69			var size uint16
    70			switch {
    71			case ch < 144:
    72				// size 8, 000110000  .. 10111111
    73				bits = ch + 48
    74				size = 8
    75				break
    76			case ch < 256:
    77				// size 9, 110010000 .. 111111111
    78				bits = ch + 400 - 144
    79				size = 9
    80				break
    81			case ch < 280:
    82				// size 7, 0000000 .. 0010111
    83				bits = ch - 256
    84				size = 7
    85				break
    86			default:
    87				// size 8, 11000000 .. 11000111
    88				bits = ch + 192 - 280
    89				size = 8
    90			}
    91			codes[ch] = hcode{code: reverseBits(bits, byte(size)), len: size}
    92		}
    93		return h
    94	}
    95	
    96	func generateFixedOffsetEncoding() *huffmanEncoder {
    97		h := newHuffmanEncoder(30)
    98		codes := h.codes
    99		for ch := range codes {
   100			codes[ch] = hcode{code: reverseBits(uint16(ch), 5), len: 5}
   101		}
   102		return h
   103	}
   104	
   105	var fixedLiteralEncoding *huffmanEncoder = generateFixedLiteralEncoding()
   106	var fixedOffsetEncoding *huffmanEncoder = generateFixedOffsetEncoding()
   107	
   108	func (h *huffmanEncoder) bitLength(freq []int32) int {
   109		var total int
   110		for i, f := range freq {
   111			if f != 0 {
   112				total += int(f) * int(h.codes[i].len)
   113			}
   114		}
   115		return total
   116	}
   117	
   118	const maxBitsLimit = 16
   119	
   120	// Return the number of literals assigned to each bit size in the Huffman encoding
   121	//
   122	// This method is only called when list.length >= 3
   123	// The cases of 0, 1, and 2 literals are handled by special case code.
   124	//
   125	// list  An array of the literals with non-zero frequencies
   126	//             and their associated frequencies. The array is in order of increasing
   127	//             frequency, and has as its last element a special element with frequency
   128	//             MaxInt32
   129	// maxBits     The maximum number of bits that should be used to encode any literal.
   130	//             Must be less than 16.
   131	// return      An integer array in which array[i] indicates the number of literals
   132	//             that should be encoded in i bits.
   133	func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 {
   134		if maxBits >= maxBitsLimit {
   135			panic("flate: maxBits too large")
   136		}
   137		n := int32(len(list))
   138		list = list[0 : n+1]
   139		list[n] = maxNode()
   140	
   141		// The tree can't have greater depth than n - 1, no matter what. This
   142		// saves a little bit of work in some small cases
   143		if maxBits > n-1 {
   144			maxBits = n - 1
   145		}
   146	
   147		// Create information about each of the levels.
   148		// A bogus "Level 0" whose sole purpose is so that
   149		// level1.prev.needed==0.  This makes level1.nextPairFreq
   150		// be a legitimate value that never gets chosen.
   151		var levels [maxBitsLimit]levelInfo
   152		// leafCounts[i] counts the number of literals at the left
   153		// of ancestors of the rightmost node at level i.
   154		// leafCounts[i][j] is the number of literals at the left
   155		// of the level j ancestor.
   156		var leafCounts [maxBitsLimit][maxBitsLimit]int32
   157	
   158		for level := int32(1); level <= maxBits; level++ {
   159			// For every level, the first two items are the first two characters.
   160			// We initialize the levels as if we had already figured this out.
   161			levels[level] = levelInfo{
   162				level:        level,
   163				lastFreq:     list[1].freq,
   164				nextCharFreq: list[2].freq,
   165				nextPairFreq: list[0].freq + list[1].freq,
   166			}
   167			leafCounts[level][level] = 2
   168			if level == 1 {
   169				levels[level].nextPairFreq = math.MaxInt32
   170			}
   171		}
   172	
   173		// We need a total of 2*n - 2 items at top level and have already generated 2.
   174		levels[maxBits].needed = 2*n - 4
   175	
   176		level := maxBits
   177		for {
   178			l := &levels[level]
   179			if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 {
   180				// We've run out of both leafs and pairs.
   181				// End all calculations for this level.
   182				// To make sure we never come back to this level or any lower level,
   183				// set nextPairFreq impossibly large.
   184				l.needed = 0
   185				levels[level+1].nextPairFreq = math.MaxInt32
   186				level++
   187				continue
   188			}
   189	
   190			prevFreq := l.lastFreq
   191			if l.nextCharFreq < l.nextPairFreq {
   192				// The next item on this row is a leaf node.
   193				n := leafCounts[level][level] + 1
   194				l.lastFreq = l.nextCharFreq
   195				// Lower leafCounts are the same of the previous node.
   196				leafCounts[level][level] = n
   197				l.nextCharFreq = list[n].freq
   198			} else {
   199				// The next item on this row is a pair from the previous row.
   200				// nextPairFreq isn't valid until we generate two
   201				// more values in the level below
   202				l.lastFreq = l.nextPairFreq
   203				// Take leaf counts from the lower level, except counts[level] remains the same.
   204				copy(leafCounts[level][:level], leafCounts[level-1][:level])
   205				levels[l.level-1].needed = 2
   206			}
   207	
   208			if l.needed--; l.needed == 0 {
   209				// We've done everything we need to do for this level.
   210				// Continue calculating one level up. Fill in nextPairFreq
   211				// of that level with the sum of the two nodes we've just calculated on
   212				// this level.
   213				if l.level == maxBits {
   214					// All done!
   215					break
   216				}
   217				levels[l.level+1].nextPairFreq = prevFreq + l.lastFreq
   218				level++
   219			} else {
   220				// If we stole from below, move down temporarily to replenish it.
   221				for levels[level-1].needed > 0 {
   222					level--
   223				}
   224			}
   225		}
   226	
   227		// Somethings is wrong if at the end, the top level is null or hasn't used
   228		// all of the leaves.
   229		if leafCounts[maxBits][maxBits] != n {
   230			panic("leafCounts[maxBits][maxBits] != n")
   231		}
   232	
   233		bitCount := h.bitCount[:maxBits+1]
   234		bits := 1
   235		counts := &leafCounts[maxBits]
   236		for level := maxBits; level > 0; level-- {
   237			// chain.leafCount gives the number of literals requiring at least "bits"
   238			// bits to encode.
   239			bitCount[bits] = counts[level] - counts[level-1]
   240			bits++
   241		}
   242		return bitCount
   243	}
   244	
   245	// Look at the leaves and assign them a bit count and an encoding as specified
   246	// in RFC 1951 3.2.2
   247	func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) {
   248		code := uint16(0)
   249		for n, bits := range bitCount {
   250			code <<= 1
   251			if n == 0 || bits == 0 {
   252				continue
   253			}
   254			// The literals list[len(list)-bits] .. list[len(list)-bits]
   255			// are encoded using "bits" bits, and get the values
   256			// code, code + 1, ....  The code values are
   257			// assigned in literal order (not frequency order).
   258			chunk := list[len(list)-int(bits):]
   259	
   260			h.lns.sort(chunk)
   261			for _, node := range chunk {
   262				h.codes[node.literal] = hcode{code: reverseBits(code, uint8(n)), len: uint16(n)}
   263				code++
   264			}
   265			list = list[0 : len(list)-int(bits)]
   266		}
   267	}
   268	
   269	// Update this Huffman Code object to be the minimum code for the specified frequency count.
   270	//
   271	// freq  An array of frequencies, in which frequency[i] gives the frequency of literal i.
   272	// maxBits  The maximum number of bits to use for any literal.
   273	func (h *huffmanEncoder) generate(freq []int32, maxBits int32) {
   274		if h.freqcache == nil {
   275			// Allocate a reusable buffer with the longest possible frequency table.
   276			// Possible lengths are codegenCodeCount, offsetCodeCount and maxNumLit.
   277			// The largest of these is maxNumLit, so we allocate for that case.
   278			h.freqcache = make([]literalNode, maxNumLit+1)
   279		}
   280		list := h.freqcache[:len(freq)+1]
   281		// Number of non-zero literals
   282		count := 0
   283		// Set list to be the set of all non-zero literals and their frequencies
   284		for i, f := range freq {
   285			if f != 0 {
   286				list[count] = literalNode{uint16(i), f}
   287				count++
   288			} else {
   289				list[count] = literalNode{}
   290				h.codes[i].len = 0
   291			}
   292		}
   293		list[len(freq)] = literalNode{}
   294	
   295		list = list[:count]
   296		if count <= 2 {
   297			// Handle the small cases here, because they are awkward for the general case code. With
   298			// two or fewer literals, everything has bit length 1.
   299			for i, node := range list {
   300				// "list" is in order of increasing literal value.
   301				h.codes[node.literal].set(uint16(i), 1)
   302			}
   303			return
   304		}
   305		h.lfs.sort(list)
   306	
   307		// Get the number of literals for each bit count
   308		bitCount := h.bitCounts(list, maxBits)
   309		// And do the assignment
   310		h.assignEncodingAndSize(bitCount, list)
   311	}
   312	
   313	type byLiteral []literalNode
   314	
   315	func (s *byLiteral) sort(a []literalNode) {
   316		*s = byLiteral(a)
   317		sort.Sort(s)
   318	}
   319	
   320	func (s byLiteral) Len() int { return len(s) }
   321	
   322	func (s byLiteral) Less(i, j int) bool {
   323		return s[i].literal < s[j].literal
   324	}
   325	
   326	func (s byLiteral) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
   327	
   328	type byFreq []literalNode
   329	
   330	func (s *byFreq) sort(a []literalNode) {
   331		*s = byFreq(a)
   332		sort.Sort(s)
   333	}
   334	
   335	func (s byFreq) Len() int { return len(s) }
   336	
   337	func (s byFreq) Less(i, j int) bool {
   338		if s[i].freq == s[j].freq {
   339			return s[i].literal < s[j].literal
   340		}
   341		return s[i].freq < s[j].freq
   342	}
   343	
   344	func (s byFreq) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
   345	

View as plain text