reinstating hpack conformance tests

.github/workflows/ci.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [2.7 ,'3.0', 3.1, 3.2, 3.3, 3.4, 4.0, jruby, truffleruby]
+        ruby: [2.7 ,'3.0', 3.1, 3.2, 3.3, 3.4, 4.0, head, jruby, truffleruby]
 
     steps:
       - uses: actions/checkout@v4
@@ -35,7 +35,7 @@ jobs:
           RUBY_VERSION=`ruby -e 'puts RUBY_VERSION'`
           RUBY_PLATFORM=`ruby -e 'puts RUBY_PLATFORM'`
           RUBY_ENGINE=`ruby -e 'puts RUBY_ENGINE'`
-          if [[ "$RUBY_ENGINE" = "ruby" ]] && [[ ${RUBY_VERSION:0:1} = "3" ]] && [[ ! $RUBYOPT =~ "jit" ]]; then
+          if [[ "$RUBY_ENGINE" = "ruby" ]] && [[ ${RUBY_VERSION:0:1} = "4" ]] && [[ ! $RUBYOPT =~ "jit" ]]; then
             echo "running runtime type checking..."
             bundle exec rbs collection install
             export RUBYOPT="-rbundler/setup -rrbs/test/setup"

.gitignore

@@ -1,3 +1,4 @@
 .idea/
 Gemfile.lock
 *.gem
+spec/hpack-test-case

Rakefile

@@ -9,16 +9,17 @@ require_relative "tasks/generate_huffman_table"
 
 RUBY_MAJOR_MINOR = RUBY_VERSION.split(".").first(2).join(".")
 
-begin
-  require "rspec/core/rake_task"
-  RSpec::Core::RakeTask.new(:spec) do |t|
-    t.exclude_pattern = "./spec/hpack_test_spec.rb"
-  end
+require "rspec/core/rake_task"
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.exclude_pattern = "./spec/hpack_test_spec.rb"
+end
 
-  RSpec::Core::RakeTask.new(:hpack) do |t|
-    t.pattern = "./spec/hpack_test_spec.rb"
-  end
-rescue LoadError
+RSpec::Core::RakeTask.new(:hpack) do |t|
+  t.pattern = "./spec/hpack_test_spec.rb"
+end
+
+task :prepare_hpack do
+  system("git clone --depth 1 https://github.com/http2jp/hpack-test-case.git spec/hpack-test-case")
 end
 
 begin
@@ -104,6 +105,6 @@ end
 
 default_tasks = %i[spec]
 default_tasks << :rubocop if defined?(RuboCop) && RUBY_ENGINE == "ruby"
-default_tasks += %i[h2spec_install h2spec] if ENV.key?("CI")
+default_tasks += %i[prepare_hpack hpack h2spec_install h2spec] if ENV.key?("CI")
 task default: default_tasks
-task all: %i[default hpack]
+task all: default_tasks

lib/http/2/header/decompressor.rb

@@ -16,6 +16,11 @@ class Decompressor
       def initialize(settings = Settings.new)
         @cc = EncodingContext.new(settings)
         @max_header_list_size = settings.settings_max_header_list_size
+        @accept_invalid_headers = false
+      end
+
+      def accept_invalid_headers!
+        @accept_invalid_headers = true
       end
 
       # Set the maximum dynamic table +size+ allowed by our acknowledged
@@ -110,12 +115,14 @@ def decode(buf, frame = nil)
             next if field.nil?
 
             is_pseudo_header = field.start_with?(":")
-            if !decoding_pseudo_headers && is_pseudo_header
+            if !decoding_pseudo_headers && is_pseudo_header && !@accept_invalid_headers
               raise ProtocolError, "one or more pseudo headers encountered after regular headers"
             end
 
             decoding_pseudo_headers = is_pseudo_header
-            raise ProtocolError, "invalid header received: #{field}" if FORBIDDEN_HEADERS.include?(field)
+            if !@accept_invalid_headers && FORBIDDEN_HEADERS.include?(field)
+              raise ProtocolError, "invalid header received: #{field}"
+            end
 
             header_list_size += field.bytesize + value.bytesize + 32
             raise ProtocolError, "header list exceeds configured maximum" if header_list_size > @max_header_list_size

lib/http/2/header/encoding_context.rb

@@ -110,11 +110,11 @@ def initialize(settings = Settings.new)
         @unshifts = 0
         @settings = settings
         # Current dynamic table size, as known to both encoder and decoder.
-        @limit =
-          # Maximum dynamic table size allowed by SETTINGS_HEADER_TABLE_SIZE.
-          @max_size_limit =
-            # Smallest maximum size set since the last dynamic table size update.
-            @lowest_max_size_limit = settings.table_size
+        @limit = settings.table_size
+        # Maximum dynamic table size allowed by SETTINGS_HEADER_TABLE_SIZE.
+        @max_size_limit =
+          # Smallest maximum size set since the last dynamic table size update.
+          @lowest_max_size_limit = MAX_HEADER_TABLE_SIZE
         @_table_updated = @_max_table_size_updated = false
         @current_table_size = 0
       end
@@ -340,7 +340,7 @@ def encode_change_table_size
         end
         @lowest_max_size_limit = @max_size_limit
 
-        max_table_size = @settings.table_size
+        max_table_size = MAX_HEADER_TABLE_SIZE
         max_table_size = @max_size_limit if @max_size_limit < max_table_size
 
         return unless max_table_size != @limit

lib/http/2/settings.rb

@@ -2,10 +2,11 @@
 
 module HTTP2
   MAX_HEADER_LIST_SIZE = (2 << 30) - 1
+  MAX_HEADER_TABLE_SIZE = 4096
 
   # Default values for SETTINGS frame, as defined by the spec.
   SPEC_DEFAULT_CONNECTION_SETTINGS = {
-    settings_header_table_size: 4096,
+    settings_header_table_size: MAX_HEADER_TABLE_SIZE,
     settings_enable_push: 1, # enabled for servers
     settings_max_concurrent_streams: Framer::MAX_STREAM_ID, # unlimited
     settings_initial_window_size: 65_535,
@@ -31,7 +32,7 @@ module HTTP2
   ) do
     def initialize(
       # connection settings
-      settings_header_table_size: 4096,
+      settings_header_table_size: MAX_HEADER_TABLE_SIZE,
       settings_enable_push: 1,
       settings_max_concurrent_streams: 100,
       settings_initial_window_size: 65_535,
@@ -40,7 +41,7 @@ def initialize(
       # encoding context settings
       huffman: :shorter,
       index: :all,
-      table_size: 4096
+      table_size: MAX_HEADER_TABLE_SIZE
     )
       super
     end

sig/header/decompressor.rbs

@@ -7,6 +7,8 @@ module HTTP2
 
       @cc: EncodingContext
 
+      @accept_invalid_headers: bool
+
       attr_writer max_header_list_size: Integer
 
       def max_table_size=: (Integer) -> void
@@ -19,6 +21,9 @@ module HTTP2
 
       def decode: (String, frame?) -> Array[header_pair]
                 | (String) -> Array[header_pair]
+
+      def accept_invalid_headers!: () -> void
+
       private
 
       def initialize: (?Settings settings) -> void

sig/settings.rbs

@@ -1,6 +1,8 @@
 module HTTP2
   MAX_HEADER_LIST_SIZE: Integer
 
+  MAX_HEADER_TABLE_SIZE: Integer
+
   SPEC_DEFAULT_CONNECTION_SETTINGS: Hash[Symbol, Integer]
 
   ENCODING_CONTEXT_SETTINGS: Array[Symbol]

spec/hpack_test_spec.rb

@@ -36,12 +36,14 @@
             story = JSON.parse(File.read("#{path}/#{file}"))
             cases = story["cases"]
             table_size = cases[0]["header_table_size"] || 4096
-            @dc = Decompressor.new(table_size: table_size)
+            settings = Settings.new(table_size: table_size)
+            @dc = Decompressor.new(settings)
+            @dc.accept_invalid_headers!
             cases.each do |c|
               wire = [c["wire"]].pack("H*").force_encoding(Encoding::BINARY)
-              @emitted = @dc.decode(HTTP2::Buffer.new(wire))
+              @emitted = @dc.decode(wire)
               headers = c["headers"].flat_map(&:to_a)
-              expect(@emitted).to eq headers
+              expect(@emitted).to match_array(headers)
             end
           end
         end
@@ -64,6 +66,7 @@
         [4096, 512].each do |table_size|
           options = { table_size: table_size }
           options.update(encoding_options)
+          settings = HTTP2::Settings.new(**options)
 
           context "with #{mode}#{huffman} mode and table_size #{table_size}" do
             path = File.expand_path("hpack-test-case/raw-data", File.dirname(__FILE__))
@@ -73,13 +76,14 @@
               it "should encode #{file}" do
                 story = JSON.parse(File.read("#{path}/#{file}"))
                 cases = story["cases"]
-                @cc = Compressor.new(options)
-                @dc = Decompressor.new(options)
+                @cc = Compressor.new(settings)
+                @dc = Decompressor.new(settings)
+                @dc.accept_invalid_headers!
                 cases.each do |c|
                   headers = c["headers"].flat_map(&:to_a)
                   wire = @cc.encode(headers)
-                  decoded = @dc.decode(HTTP2::Buffer.new(wire))
-                  expect(decoded).to eq headers
+                  decoded = @dc.decode(wire)
+                  expect(decoded).to match_array(headers)
                 end
               end
             end