From ab6a0765d24185bc296db0e962777438472b3706 Mon Sep 17 00:00:00 2001
From: Stephen Paul Weber <singpolyma@singpolyma.net>
Date: Thu, 14 Mar 2019 20:44:36 -0500
Subject: [PATCH] Binary encoding

Every AST node uses as_json to express the generic structure for
encoding itself.  Expression aliases as_cbor to as_json by default and
then uses as_cbor in to_cbor (aliased as to_binary).

This design is because we need to override the encoding of Double for
CBOR, but want to keep the normal Float value for as_json in case that
is used directly (to avoid surprising the user).
---
 lib/dhall/ast.rb           | 215 ++++++++++++++++++++++++++++++++++++-
 lib/dhall/binary.rb        |  28 ++---
 lib/dhall/builtins.rb      |   4 +
 test/test_as_json.rb       |  23 ++++
 test/test_binary.rb        |  13 ++-
 test/test_normalization.rb |   4 +-
 test/test_resolve.rb       |  28 ++---
 7 files changed, 274 insertions(+), 41 deletions(-)
 create mode 100644 test/test_as_json.rb

diff --git a/lib/dhall/ast.rb b/lib/dhall/ast.rb
index 2bb6d0e..7d5e231 100644
--- a/lib/dhall/ast.rb
+++ b/lib/dhall/ast.rb
@@ -110,6 +110,10 @@ module Dhall
 			function Expression
 			arguments Util::ArrayOf.new(Expression, min: 1)
 		end)
+
+		def as_json
+			[0, function.as_json, *arguments.map(&:as_json)]
+		end
 	end
 
 	class Function < Expression
@@ -137,9 +141,25 @@ module Dhall
 				args.first.shift(1, var, 0)
 			).shift(-1, var, 0).normalize
 		end
+
+		def as_json
+			if var == "_"
+				[1, type.as_json, body.as_json]
+			else
+				[1, var, type.as_json, body.as_json]
+			end
+		end
 	end
 
-	class Forall < Function; end
+	class Forall < Function
+		def as_json
+			if var == "_"
+				[2, type.as_json, body.as_json]
+			else
+				[2, var, type.as_json, body.as_json]
+			end
+		end
+	end
 
 	class Bool < Expression
 		include(ValueSemantics.for_attributes do
@@ -161,6 +181,10 @@ module Dhall
 		def dhall_eq(other)
 			reduce(other, super)
 		end
+
+		def as_json
+			value
+		end
 	end
 
 	class Variable < Expression
@@ -172,6 +196,16 @@ module Dhall
 		def self.[](name, index=0)
 			new(name: name, index: index)
 		end
+
+		def as_json
+			if name == "_"
+				index
+			elsif index == 0
+				name
+			else
+				[name, index]
+			end
+		end
 	end
 
 	class Operator < Expression
@@ -180,6 +214,10 @@ module Dhall
 			rhs Expression
 		end)
 
+		def as_json
+			[3, OPERATORS.index(self.class), lhs.as_json, rhs.as_json]
+		end
+
 		class Or < Operator; end
 		class And < Operator; end
 		class Equal < Operator; end
@@ -192,6 +230,14 @@ module Dhall
 		class RightBiasedRecordMerge < Operator; end
 		class RecursiveRecordTypeMerge < Operator; end
 		class ImportFallback < Operator; end
+
+		OPERATORS = [
+			Or, And, Equal, NotEqual,
+			Plus, Times,
+			TextConcatenate, ListConcatenate,
+			RecursiveRecordMerge, RightBiasedRecordMerge, RecursiveRecordTypeMerge,
+			ImportFallback
+		].freeze
 	end
 
 	class List < Expression
@@ -203,6 +249,10 @@ module Dhall
 			List.new(elements: args)
 		end
 
+		def as_json
+			[4, nil, *elements.map(&:as_json)]
+		end
+
 		def type
 			# TODO: inferred element type
 		end
@@ -245,6 +295,10 @@ module Dhall
 			type Expression
 		end)
 
+		def as_json
+			[4, type.as_json]
+		end
+
 		def map(type: nil)
 			type.nil? ? self : with(type: type)
 		end
@@ -283,6 +337,10 @@ module Dhall
 		def reduce(_, &block)
 			block[value]
 		end
+
+		def as_json
+			[5, type&.as_json, value.as_json]
+		end
 	end
 
 	class OptionalNone < Optional
@@ -293,6 +351,10 @@ module Dhall
 		def reduce(z)
 			z
 		end
+
+		def as_json
+			[5, type.as_json]
+		end
 	end
 
 	class Merge < Expression
@@ -301,6 +363,11 @@ module Dhall
 			input  Expression
 			type   Either(Expression, nil)
 		end)
+
+		def as_json
+			[6, record.as_json, input.as_json] +
+				(type.nil? ? [] : [type.as_json])
+		end
 	end
 
 	class RecordType < Expression
@@ -323,6 +390,10 @@ module Dhall
 		def eql?(other)
 			self == other
 		end
+
+		def as_json
+			[7, Hash[record.to_a.map { |k, v| [k, v.as_json] }.sort]]
+		end
 	end
 
 	class EmptyRecordType < Expression
@@ -331,6 +402,10 @@ module Dhall
 		def deep_merge_type(other)
 			other
 		end
+
+		def as_json
+			[7, {}]
+		end
 	end
 
 	class Record < Expression
@@ -371,6 +446,10 @@ module Dhall
 		def eql?(other)
 			self == other
 		end
+
+		def as_json
+			[8, Hash[record.to_a.map { |k, v| [k, v.as_json] }.sort]]
+		end
 	end
 
 	class EmptyRecord < Expression
@@ -391,6 +470,10 @@ module Dhall
 		def merge(other)
 			other
 		end
+
+		def as_json
+			[8, {}]
+		end
 	end
 
 	class RecordSelection < Expression
@@ -398,6 +481,10 @@ module Dhall
 			record Expression
 			selector ::String
 		end)
+
+		def as_json
+			[9, record.as_json, selector]
+		end
 	end
 
 	class RecordProjection < Expression
@@ -405,12 +492,20 @@ module Dhall
 			record Expression
 			selectors Util::ArrayOf.new(::String, min: 1)
 		end)
+
+		def as_json
+			[10, record.as_json, *selectors]
+		end
 	end
 
 	class EmptyRecordProjection < Expression
 		include(ValueSemantics.for_attributes do
 			record Expression
 		end)
+
+		def as_json
+			[10, record.as_json]
+		end
 	end
 
 	class UnionType < Expression
@@ -438,6 +533,10 @@ module Dhall
 				)
 			)
 		end
+
+		def as_json
+			[11, Hash[alternatives.to_a.map { |k, v| [k, v.as_json] }.sort]]
+		end
 	end
 
 	class Union < Expression
@@ -446,6 +545,10 @@ module Dhall
 			value        Expression
 			alternatives UnionType
 		end)
+
+		def as_json
+			[12, tag, value.as_json, alternatives.as_json.last]
+		end
 	end
 
 	class If < Expression
@@ -454,6 +557,10 @@ module Dhall
 			self.then Expression
 			self.else Expression
 		end)
+
+		def as_json
+			[14, predicate.as_json, self.then.as_json, self.else.as_json]
+		end
 	end
 
 	class Number < Expression
@@ -500,6 +607,10 @@ module Dhall
 		def pred
 			with(value: [0, value - 1].max)
 		end
+
+		def as_json
+			[15, value]
+		end
 	end
 
 	class Integer < Number
@@ -510,6 +621,10 @@ module Dhall
 		def to_s
 			"#{value >= 0 ? "+" : ""}#{value}"
 		end
+
+		def as_json
+			[16, value]
+		end
 	end
 
 	class Double < Number
@@ -520,6 +635,34 @@ module Dhall
 		def to_s
 			value.to_s
 		end
+
+		def single?
+			[value].pack("g").unpack("g").first == value
+		end
+
+		def as_json
+			value
+		end
+
+		def as_cbor
+			self
+		end
+
+		def to_cbor(packer=nil)
+			if [0.0, Float::INFINITY, -Float::INFINITY].include?(value) ||
+			   value.nan?
+				return value.to_cbor(packer)
+			end
+
+			# Dhall spec requires *not* using half-precision CBOR floats
+			bytes = single? ? [0xFA, value].pack("Cg") : [0xFB, value].pack("CG")
+			if packer
+				packer.buffer.write(bytes)
+				packer
+			else
+				bytes
+			end
+		end
 	end
 
 	class Text < Expression
@@ -534,12 +677,21 @@ module Dhall
 				super
 			end
 		end
+
+		def as_json
+			[18, value]
+		end
 	end
 
 	class TextLiteral < Expression
 		include(ValueSemantics.for_attributes do
 			chunks ArrayOf(Expression)
 		end)
+
+		def as_json
+			raise "TextLiteral must start with a Text" unless chunks.first.is_a?(Text)
+			[18, *chunks.map { |chunk| chunk.is_a?(Text) ? chunk.value : chunk.as_json }]
+		end
 	end
 
 	class Import < Expression
@@ -549,6 +701,16 @@ module Dhall
 			@path = path
 		end
 
+		def as_json
+			[
+				24,
+				@integrity_check&.as_json,
+				IMPORT_TYPES.index(@import_type),
+				PATH_TYPES.index(@path.class),
+				*@path.as_json
+			]
+		end
+
 		class URI
 			include(ValueSemantics.for_attributes do
 				headers   Either(nil, Expression)
@@ -581,6 +743,10 @@ module Dhall
 			def uri
 				URI("#{scheme}://#{authority}/#{path.join("/")}?#{query}")
 			end
+
+			def as_json
+				[headers.as_json, authority, *path, query, fragment]
+			end
 		end
 
 		class Http < URI
@@ -630,6 +796,10 @@ module Dhall
 			def to_s
 				pathname.to_s
 			end
+
+			def as_json
+				path
+			end
 		end
 
 		class AbsolutePath < Path
@@ -677,12 +847,20 @@ module Dhall
 					Path.from_string(val)
 				end.resolve(resolver)
 			end
+
+			def as_json
+				var
+			end
 		end
 
 		class MissingImport
 			def resolve(*)
 				Promise.new.reject(ImportFailedException.new("missing"))
 			end
+
+			def as_json
+				[]
+			end
 		end
 
 		class IntegrityCheck
@@ -691,6 +869,29 @@ module Dhall
 				@data = data
 			end
 		end
+
+		class Expression
+			def self.call(import_value)
+				Dhall.from_binary(import_value)
+			end
+		end
+
+		class Text
+			def self.call(import_value)
+				Dhall::Text.new(value: import_value)
+			end
+		end
+
+		IMPORT_TYPES = [
+			Expression,
+			Text
+		].freeze
+
+		PATH_TYPES = [
+			Http, Https,
+			AbsolutePath, RelativePath, RelativeToParentPath, RelativeToHomePath,
+			EnvironmentVariable, MissingImport
+		].freeze
 	end
 
 	class Let < Expression
@@ -699,6 +900,10 @@ module Dhall
 			assign Expression
 			type   Either(nil, Expression)
 		end)
+
+		def as_json
+			[var, type&.as_json, assign.as_json]
+		end
 	end
 
 	class LetBlock < Expression
@@ -706,6 +911,10 @@ module Dhall
 			lets ArrayOf(Let)
 			body Expression
 		end)
+
+		def as_json
+			[25, *lets.flat_map(&:as_json), body.as_json]
+		end
 	end
 
 	class TypeAnnotation < Expression
@@ -713,5 +922,9 @@ module Dhall
 			value Expression
 			type  Expression
 		end)
+
+		def as_json
+			[26, value.as_json, type.as_json]
+		end
 	end
 end
diff --git a/lib/dhall/binary.rb b/lib/dhall/binary.rb
index 89f63e9..0c09099 100644
--- a/lib/dhall/binary.rb
+++ b/lib/dhall/binary.rb
@@ -24,6 +24,15 @@ module Dhall
 
 			new(*args)
 		end
+
+		def to_cbor(io=nil)
+			CBOR.encode(as_cbor, io)
+		end
+		alias to_binary to_cbor
+
+		def as_cbor
+			as_json
+		end
 	end
 
 	class Application
@@ -51,14 +60,6 @@ module Dhall
 	end
 
 	class Operator
-		OPERATORS = [
-			Or, And, Equal, NotEqual,
-			Plus, Times,
-			TextConcatenate, ListConcatenate,
-			RecursiveRecordMerge, RightBiasedRecordMerge, RecursiveRecordTypeMerge,
-			ImportFallback
-		].freeze
-
 		def self.decode(opcode, lhs, rhs)
 			OPERATORS[opcode].new(
 				lhs: Dhall.decode(lhs),
@@ -176,17 +177,6 @@ module Dhall
 	end
 
 	class Import
-		IMPORT_TYPES = [
-			Dhall.method(:from_binary),
-			->(x) { Text.new(value: x) }
-		].freeze
-
-		PATH_TYPES = [
-			Http, Https,
-			AbsolutePath, RelativePath, RelativeToParentPath, RelativeToHomePath,
-			EnvironmentVariable, MissingImport
-		].freeze
-
 		def self.decode(integrity_check, import_type, path_type, *parts)
 			parts[0] = Dhall.decode(parts[0]) if path_type < 2 && !parts[0].nil?
 
diff --git a/lib/dhall/builtins.rb b/lib/dhall/builtins.rb
index c065720..18ed7d7 100644
--- a/lib/dhall/builtins.rb
+++ b/lib/dhall/builtins.rb
@@ -13,6 +13,10 @@ module Dhall
 			end
 		end
 
+		def as_json
+			self.class.name.split(/::/).last.gsub(/_/, "/")
+		end
+
 		protected
 
 		def attributes
diff --git a/test/test_as_json.rb b/test/test_as_json.rb
new file mode 100644
index 0000000..a3bc7d6
--- /dev/null
+++ b/test/test_as_json.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "minitest/autorun"
+require "pathname"
+require "cbor"
+
+require "dhall/ast"
+require "dhall/binary"
+
+class TestAsJson < Minitest::Test
+	DIRPATH = Pathname.new(File.dirname(__FILE__))
+	TESTS = DIRPATH + "normalization/"
+
+	Pathname.glob(TESTS + "**/*.dhallb").each do |path|
+		test = path.relative_path_from(TESTS).to_s.sub(/.dhallb$/, "")
+		define_method("test_#{test}") do
+			assert_equal(
+				CBOR.decode(path.read).inspect,
+				Dhall.from_binary(path.read).as_json.inspect
+			)
+		end
+	end
+end
diff --git a/test/test_binary.rb b/test/test_binary.rb
index a69f79b..015da1b 100644
--- a/test/test_binary.rb
+++ b/test/test_binary.rb
@@ -6,14 +6,17 @@ require "pathname"
 require "dhall/ast"
 require "dhall/binary"
 
-class TestParser < Minitest::Test
+class TestBinary < Minitest::Test
 	DIRPATH = Pathname.new(File.dirname(__FILE__))
-	TESTS = DIRPATH + "../dhall-lang/tests/parser/success/"
+	TESTS = DIRPATH + "normalization/"
 
-	Pathname.glob(TESTS + "*B.dhallb").each do |path|
-		test = path.basename("B.dhallb").to_s
+	Pathname.glob(TESTS + "**/*.dhallb").each do |path|
+		test = path.relative_path_from(TESTS).to_s.sub(/.dhallb$/, "")
 		define_method("test_#{test}") do
-			assert_kind_of Dhall::Expression, Dhall.from_binary(path.read)
+			assert_equal(
+				path.binread,
+				Dhall.from_binary(path.binread).to_binary
+			)
 		end
 	end
 end
diff --git a/test/test_normalization.rb b/test/test_normalization.rb
index 5dd5f6b..3746083 100644
--- a/test/test_normalization.rb
+++ b/test/test_normalization.rb
@@ -20,8 +20,8 @@ class TestNormalization < Minitest::Test
 		define_method("test_#{test.gsub(/\//, "_")}") do
 			Dhall::Function.disable_alpha_normalization! if test =~ /^standard\//
 			assert_equal(
-				Dhall.from_binary(TESTS + "#{test}B.dhallb"),
-				Dhall.from_binary(path.read).normalize
+				Dhall.from_binary((TESTS + "#{test}B.dhallb").binread),
+				Dhall.from_binary(path.binread).normalize
 			)
 			Dhall::Function.enable_alpha_normalization! if test =~ /^standard\//
 		end
diff --git a/test/test_resolve.rb b/test/test_resolve.rb
index 8b16380..f458cfc 100644
--- a/test/test_resolve.rb
+++ b/test/test_resolve.rb
@@ -43,7 +43,7 @@ class TestResolve < Minitest::Test
 	def test_import_as_text
 		expr = Dhall::Import.new(
 			nil,
-			->(x) { Dhall::Text.new(value: x) },
+			Dhall::Import::Text,
 			Dhall::Import::RelativePath.new("text")
 		)
 
@@ -58,7 +58,7 @@ class TestResolve < Minitest::Test
 			Dhall::Variable["Natural"],
 			body: Dhall::Import.new(
 				nil,
-				Dhall.method(:from_binary),
+				Dhall::Import::Expression,
 				Dhall::Import::RelativePath.new("var")
 			)
 		)
@@ -74,7 +74,7 @@ class TestResolve < Minitest::Test
 			Dhall::Variable["Natural"],
 			body: Dhall::Import.new(
 				nil,
-				Dhall.method(:from_binary),
+				Dhall::Import::Expression,
 				Dhall::Import::RelativePath.new("import")
 			)
 		)
@@ -90,7 +90,7 @@ class TestResolve < Minitest::Test
 			Dhall::Variable["Natural"],
 			body: Dhall::Import.new(
 				nil,
-				Dhall.method(:from_binary),
+				Dhall::Import::Expression,
 				Dhall::Import::RelativePath.new("self")
 			)
 		)
@@ -105,7 +105,7 @@ class TestResolve < Minitest::Test
 			Dhall::Variable["Natural"],
 			body: Dhall::Import.new(
 				nil,
-				Dhall.method(:from_binary),
+				Dhall::Import::Expression,
 				Dhall::Import::RelativePath.new("a")
 			)
 		)
@@ -118,7 +118,7 @@ class TestResolve < Minitest::Test
 	def test_two_references_no_loop
 		expr = Dhall::Import.new(
 			nil,
-			Dhall.method(:from_binary),
+			Dhall::Import::Expression,
 			Dhall::Import::RelativePath.new("2text")
 		)
 
@@ -134,7 +134,7 @@ class TestResolve < Minitest::Test
 	def test_missing
 		expr = Dhall::Import.new(
 			nil,
-			Dhall.method(:from_binary),
+			Dhall::Import::Expression,
 			Dhall::Import::MissingImport.new
 		)
 
@@ -147,7 +147,7 @@ class TestResolve < Minitest::Test
 		expr = Dhall::Operator::ImportFallback.new(
 			lhs: Dhall::Import.new(
 				nil,
-				Dhall.method(:from_binary),
+				Dhall::Import::Expression,
 				Dhall::Import::MissingImport.new
 			),
 			rhs: Dhall::Variable["fallback"]
@@ -160,12 +160,12 @@ class TestResolve < Minitest::Test
 		expr = Dhall::Operator::ImportFallback.new(
 			lhs: Dhall::Import.new(
 				nil,
-				Dhall.method(:from_binary),
+				Dhall::Import::Expression,
 				Dhall::Import::MissingImport.new
 			),
 			rhs: Dhall::Import.new(
 				nil,
-				Dhall.method(:from_binary),
+				Dhall::Import::Expression,
 				Dhall::Import::RelativePath.new("import")
 			)
 		)
@@ -179,7 +179,7 @@ class TestResolve < Minitest::Test
 
 		expr = Dhall::Import.new(
 			nil,
-			Dhall.method(:from_binary),
+			Dhall::Import::Expression,
 			Dhall::Import::AbsolutePath.new("ipfs", "TESTCID")
 		)
 
@@ -195,7 +195,7 @@ class TestResolve < Minitest::Test
 
 		expr = Dhall::Import.new(
 			nil,
-			Dhall.method(:from_binary),
+			Dhall::Import::Expression,
 			Dhall::Import::AbsolutePath.new("ipfs", "TESTCID")
 		)
 
@@ -211,8 +211,8 @@ class TestResolve < Minitest::Test
 		define_method("test_#{test.gsub(/\//, "_")}") do
 			Dhall::Function.disable_alpha_normalization! if test =~ /^standard\//
 			assert_equal(
-				Dhall.from_binary(TESTS + "#{test}B.dhallb"),
-				Dhall.from_binary(path.read).resolve.sync.normalize
+				Dhall.from_binary((TESTS + "#{test}B.dhallb").binread),
+				Dhall.from_binary(path.binread).resolve.sync.normalize
 			)
 			Dhall::Function.enable_alpha_normalization! if test =~ /^standard\//
 		end
-- 
2.34.5