6 files changed, 320 insertions(+), 84 deletions(-)

M lib/dhall.rb
M lib/dhall/ast.rb
M lib/dhall/normalize.rb
M lib/dhall/parser.rb
M lib/dhall/resolve.rb
M test/test_resolve.rb

@@ 4,7 4,7 @@ module Dhall
 	def self.load_raw(source)
 		begin
 			return from_binary(source) if source.encoding == Encoding::BINARY
-		rescue
+		rescue Exception # rubocop:disable Lint/RescueException
 			# Parsing CBOR failed, so guess this is source text in standard UTF-8
 			return load_raw(source.force_encoding("UTF-8"))
 		end

@@ 805,6 805,10 @@ module Dhall
 			[Double.new(value: other.to_f), self]
 		end
 
+		def eql?(other)
+			other.is_a?(Double) && to_cbor == other.to_cbor
+		end
+
 		def single?
 			[value].pack("g").unpack("g").first == value
 		end


@@ 898,11 902,12 @@ module Dhall
 			end
 
 			def check(expr)
-				if @protocol != :nocheck && expr.cache_key != to_s
-					raise FailureException, "#{expr} does not match #{self}"
-				end
+				return expr if @protocol == :nocheck
+
+				expr = expr.normalize
+				return expr if expr.cache_key == to_s
 
-				expr
+				raise FailureException, "#{expr} does not match #{self}"
 			end
 
 			def as_json


@@ 936,10 941,10 @@ module Dhall
 
 			def with(hash)
 				self.class.new(
-					hash.fetch(:headers),
-					authority,
-					*path,
-					query
+					hash.fetch(:headers, headers),
+					hash.fetch(:authority, authority),
+					*hash.fetch(:path, path),
+					hash.fetch(:query, query)
 				)
 			end
 


@@ 958,7 963,35 @@ module Dhall
 			end
 
 			def uri
-				URI("#{scheme}://#{authority}/#{path.join("/")}?#{query}")
+				escaped_path = path.map do |c|
+					::URI.encode_www_form_component(c).gsub("+", "%20")
+				end
+				URI("#{scheme}://#{authority}/#{escaped_path.join("/")}?#{query}")
+			end
+
+			def chain_onto(relative_to)
+				if headers.is_a?(Import)
+					with(headers: headers.with(path: headers.real_path(relative_to)))
+				else
+					self
+				end
+			end
+
+			def canonical
+				with(
+					path: (path[1..-1] + [""])
+					.reduce([[], path.first]) { |(pth, prev), c|
+						c == ".." ? [pth, prev] : [pth + [prev], c]
+					}.first.reject { |c| c == "." }
+				)
+			end
+
+			def origin
+				"#{scheme}://#{authority}"
+			end
+
+			def to_s
+				uri.to_s
 			end
 
 			def as_json


@@ 995,8 1028,12 @@ module Dhall
 				super(path: path)
 			end
 
+			def with(path:)
+				self.class.new(*path)
+			end
+
 			def self.from_string(s)
-				parts = s.split(/\//)
+				parts = s.to_s.split(/\//)
 				if parts.first == ""
 					AbsolutePath.new(*parts[1..-1])
 				elsif parts.first == "~"


@@ 1006,10 1043,18 @@ module Dhall
 				end
 			end
 
+			def canonical
+				self.class.from_string(pathname.cleanpath)
+			end
+
 			def resolve(resolver)
 				resolver.resolve_path(self)
 			end
 
+			def origin
+				"localhost"
+			end
+
 			def to_s
 				pathname.to_s
 			end


@@ 1027,24 1072,52 @@ module Dhall
 			def to_uri(scheme, authority)
 				scheme.new(nil, authority, *path, nil)
 			end
+
+			def chain_onto(relative_to)
+				if relative_to.is_a?(URI)
+					raise ImportBannedException, "remote import cannot import #{self}"
+				end
+
+				self
+			end
 		end
 
 		class RelativePath < Path
 			def pathname
 				Pathname.new(".").join(*path)
 			end
+
+			def chain_onto(relative_to)
+				relative_to.with(
+					path: relative_to.path[0..-2] + path
+				)
+			end
 		end
 
 		class RelativeToParentPath < Path
 			def pathname
 				Pathname.new("..").join(*path)
 			end
+
+			def chain_onto(relative_to)
+				relative_to.with(
+					path: relative_to.path[0..-2] + [".."] + path
+				)
+			end
 		end
 
 		class RelativeToHomePath < Path
 			def pathname
 				Pathname.new("~").join(*@path)
 			end
+
+			def chain_onto(*)
+				if relative_to.is_a?(URI)
+					raise ImportBannedException, "remote import cannot import #{self}"
+				end
+
+				self
+			end
 		end
 
 		class EnvironmentVariable


@@ 1070,23 1143,43 @@ module Dhall
 				@var = var
 			end
 
-			def value
-				ENV.fetch(@var)
+			def chain_onto(relative_to)
+				if relative_to.is_a?(URI)
+					raise ImportBannedException, "remote import cannot import #{self}"
+				end
+
+				real_path.chain_onto(relative_to)
+			end
+
+			def canonical
+				real_path.canonical
+			end
+
+			def real_path
+				val = ENV.fetch(@var) do
+					raise ImportFailedException, "No #{self}"
+				end
+				if val =~ /\Ahttps?:\/\//
+					URI.from_uri(URI(val))
+				else
+					Path.from_string(val)
+				end
 			end
 
 			def resolve(resolver)
 				Promise.resolve(nil).then do
-					val = ENV.fetch(@var) do
-						raise ImportFailedException, "No ENV #{@var}"
-					end
-					if val =~ /\Ahttps?:\/\//
-						URI.from_uri(URI(value))
-					else
-						Path.from_string(val)
-					end.resolve(resolver)
+					real_path.resolve(resolver)
 				end
 			end
 
+			def origin
+				"localhost"
+			end
+
+			def to_s
+				"env:#{as_json}"
+			end
+
 			def as_json
 				@var.gsub(/[\"\\\a\b\f\n\r\t\v]/) do |c|
 					"\\" + ESCAPES.find { |(_, v)| v == c }.first


@@ 1095,10 1188,24 @@ module Dhall
 		end
 
 		class MissingImport
+			def chain_onto(*)
+				self
+			end
+
+			def canonical
+				self
+			end
+
 			def resolve(*)
 				Promise.new.reject(ImportFailedException.new("missing"))
 			end
 
+			def origin; end
+
+			def to_s
+				"missing"
+			end
+
 			def as_json
 				[]
 			end


@@ 1141,6 1248,30 @@ module Dhall
 			)
 		end
 
+		def with(options)
+			self.class.new(
+				options.fetch(:integrity_check, integrity_check),
+				options.fetch(:import_type, import_type),
+				options.fetch(:path, path)
+			)
+		end
+
+		def real_path(relative_to)
+			path.chain_onto(relative_to).canonical
+		end
+
+		def parse_and_check(raw)
+			integrity_check.check(import_type.call(raw))
+		end
+
+		def cache_key(relative_to)
+			if integrity_check.protocol == :nocheck
+				real_path(relative_to).to_s
+			else
+				integrity_check.to_s
+			end
+		end
+
 		def as_json
 			[
 				24,

@@ 18,7 18,7 @@ module Dhall
 					x.map(&block)
 				end,
 				ExpressionHash                => lambda do |x|
-					Hash[x.map { |k, v| [k, v.nil? ? v : block[v]] }]
+					Hash[x.map { |k, v| [k, v.nil? ? v : block[v]] }.sort]
 				end
 			)
 		end

@@ 497,19 497,24 @@ module Dhall
 				"https" => Dhall::Import::Https
 			}.freeze
 
-			def self.escape(s)
-				URI.encode_www_form_component(s).gsub("+", "%20")
-			end
-
 			def value
 				http = capture(:http_raw)
 				SCHEME.fetch(http.capture(:scheme).value).new(
 					capture(:import_hashed)&.value(Dhall::Import::Expression),
 					http.capture(:authority).value,
-					*http.capture(:path).captures(:path_component).map(&:value),
+					*unescaped_components,
 					http.capture(:query)&.value
 				)
 			end
+
+			def unescaped_components
+				capture(:http_raw)
+					.capture(:path)
+					.captures(:path_component)
+					.map do |pc|
+						pc.value(URI.method(:unescape))
+					end
+			end
 		end
 
 		module Env


@@ 575,11 580,11 @@ module Dhall
 		end
 
 		module PathComponent
-			def value(escaper=:itself.to_proc)
+			def value(unescaper=:itself.to_proc)
 				if captures.key?(:quoted_path_component)
-					escaper.call(capture(:quoted_path_component).value)
+					capture(:quoted_path_component).value
 				else
-					capture(:unquoted_path_component).value
+					unescaper.call(capture(:unquoted_path_component).value)
 				end
 			end
 		end

@@ 1,7 1,8 @@
 # frozen_string_literal: true
 
-require "set"
+require "pathname"
 require "promise.rb"
+require "set"
 
 require "dhall/ast"
 require "dhall/binary"


@@ 15,13 16,36 @@ module Dhall
 	module Resolvers
 		ReadPathSources = lambda do |sources|
 			sources.map do |source|
-				Promise.resolve(nil).then { source.pathname.read }
+				Promise.resolve(nil).then { source.pathname.binread }
 			end
 		end
 
-		ReadHttpSources = lambda do |sources|
+		PreflightCORS = lambda do |source, parent_origin|
+			uri = source.uri
+			if parent_origin != "localhost" && parent_origin != source.origin
+				req = Net::HTTP::Options.new(uri)
+				req["Origin"] = parent_origin
+				req["Access-Control-Request-Method"] = "GET"
+				req["Access-Control-Request-Headers"] =
+					source.headers.map { |h| h.fetch("header").to_s }.join(",")
+				r = Net::HTTP.start(
+					uri.hostname,
+					uri.port,
+					use_ssl: uri.scheme == "https"
+				) { |http| http.request(req) }
+
+				raise ImportFailedException, source if r.code != "200"
+				unless r["Access-Control-Allow-Origin"] == parent_origin ||
+				       r["Access-Control-Allow-Origin"] == "*"
+					raise ImportBannedException, source
+				end
+			end
+		end
+
+		ReadHttpSources = lambda do |sources, parent_origin|
 			sources.map do |source|
 				Promise.resolve(nil).then do
+					PreflightCORS.call(source, parent_origin)
 					uri = source.uri
 					req = Net::HTTP::Get.new(uri)
 					source.headers.each do |header|


@@ 58,6 82,10 @@ module Dhall
 				@public_gateway = public_gateway
 			end
 
+			def arity
+				1
+			end
+
 			def call(sources)
 				@path_reader.call(sources).map.with_index do |promise, idx|
 					source = sources[idx]


@@ 80,21 108,19 @@ module Dhall
 				promise.catch {
 					@http_reader.call([
 						source.to_uri(Import::Http, "localhost:8000")
-					]).first
+					], "localhost").first
 				}.catch do
 					@https_reader.call([
 						source.to_uri(Import::Https, @public_gateway)
-					]).first
+					], "localhost").first
 				end
 			end
 		end
 
 		class ResolutionSet
-			attr_reader :reader
-
 			def initialize(reader)
 				@reader = reader
-				@parents = Set.new
+				@parents = []
 				@set = Hash.new { |h, k| h[k] = [] }
 			end
 


@@ 113,6 139,16 @@ module Dhall
 				[Array(sources), Array(promises)]
 			end
 
+			def reader
+				lambda do |sources|
+					if @reader.arity == 2
+						@reader.call(sources, @parents.last&.origin || "localhost")
+					else
+						@reader.call(sources)
+					end
+				end
+			end
+
 			def child(parent_source)
 				dup.tap do |c|
 					c.instance_eval do


@@ 132,6 168,15 @@ module Dhall
 				@path_resolutions = ResolutionSet.new(path_reader)
 				@http_resolutions = ResolutionSet.new(http_reader)
 				@https_resolutions = ResolutionSet.new(https_reader)
+				@cache = {}
+			end
+
+			def cache_fetch(key, &fallback)
+				@cache.fetch(key) do
+					Promise.resolve(nil).then(&fallback).then do |result|
+						@cache[key] = result
+					end
+				end
 			end
 
 			def resolve_path(path_source)


@@ 139,9 184,10 @@ module Dhall
 			end
 
 			def resolve_http(http_source)
-				ExpressionResolver
-					.for(http_source.headers)
-					.resolve(self).then do |headers|
+				http_source.headers.resolve(
+					resolver:    self,
+					relative_to: Dhall::Import::RelativePath.new
+				).then do |headers|
 					@http_resolutions.register(
 						http_source.with(headers: headers.normalize)
 					)


@@ 149,9 195,10 @@ module Dhall
 			end
 
 			def resolve_https(https_source)
-				ExpressionResolver
-					.for(https_source.headers)
-					.resolve(self).then do |headers|
+				https_source.headers.resolve(
+					resolver:    self,
+					relative_to: Dhall::Import::RelativePath.new
+				).then do |headers|
 					@https_resolutions.register(
 						https_source.with(headers: headers.normalize)
 					)


@@ 236,10 283,10 @@ module Dhall
 			@expr = expr
 		end
 
-		def resolve(resolver)
+		def resolve(**kwargs)
 			Util.promise_all_hash(
 				@expr.to_h.each_with_object({}) { |(attr, value), h|
-					h[attr] = ExpressionResolver.for(value).resolve(resolver)
+					h[attr] = ExpressionResolver.for(value).resolve(**kwargs)
 				}
 			).then { |h| @expr.with(h) }
 		end


@@ 247,39 294,54 @@ module Dhall
 		class ImportResolver < ExpressionResolver
 			register_for Import
 
-			def resolve(resolver)
-				@expr.instance_eval do
-					@path.resolve(resolver).then do |result|
-						@integrity_check.check(
-							@import_type.call(result)
-						).resolve(resolver.child(@path))
+			def resolve(resolver:, relative_to:)
+				Promise.resolve(nil).then do
+					resolver.cache_fetch(@expr.cache_key(relative_to)) do
+						resolve_raw(resolver: resolver, relative_to: relative_to)
 					end
 				end
 			end
+
+			def resolve_raw(resolver:, relative_to:)
+				real_path = @expr.real_path(relative_to)
+				real_path.resolve(resolver).then do |result|
+					@expr.parse_and_check(result).resolve(
+						resolver:    resolver.child(real_path),
+						relative_to: real_path
+					)
+				end
+			end
 		end
 
 		class FallbackResolver < ExpressionResolver
 			register_for Operator::ImportFallback
 
-			def resolve(resolver)
-				ExpressionResolver.for(@expr.lhs).resolve(resolver).catch do
-					ExpressionResolver.for(@expr.rhs).resolve(resolver)
+			def resolve(**kwargs)
+				ExpressionResolver.for(@expr.lhs).resolve(**kwargs).catch do
+					ExpressionResolver.for(@expr.rhs).resolve(**kwargs)
 				end
 			end
 		end
 
 		class ArrayResolver < ExpressionResolver
-			def resolve(resolver)
+			register_for Util::ArrayOf.new(Expression)
+
+			def resolve(**kwargs)
 				Promise.all(
-					@expr.map { |e| ExpressionResolver.for(e).resolve(resolver) }
+					@expr.map { |e| ExpressionResolver.for(e).resolve(**kwargs) }
 				)
 			end
 		end
 
 		class HashResolver < ExpressionResolver
-			def resolve(resolver)
+			register_for Util::HashOf.new(
+				ValueSemantics::Anything,
+				ValueSemantics::Either.new([Expression, nil])
+			)
+
+			def resolve(**kwargs)
 				Util.promise_all_hash(Hash[@expr.map do |k, v|
-					[k, ExpressionResolver.for(v).resolve(resolver)]
+					[k, ExpressionResolver.for(v).resolve(**kwargs)]
 				end])
 			end
 		end


@@ 296,8 358,14 @@ module Dhall
 	end
 
 	class Expression
-		def resolve(resolver=Resolvers::Default.new)
-			p = ExpressionResolver.for(self).resolve(resolver)
+		def resolve(
+			resolver: Resolvers::Default.new,
+			relative_to: Import::Path.from_string(Pathname.pwd + "file")
+		)
+			p = ExpressionResolver.for(self).resolve(
+				resolver:    resolver,
+				relative_to: relative_to
+			)
 			resolver.finish!
 			p
 		end

@@ 8,6 8,7 @@ require "dhall"
 
 class TestResolve < Minitest::Test
 	def setup
+		@relative_to = Dhall::Import::RelativePath.new
 		@resolver = Dhall::Resolvers::Default.new(
 			path_reader: lambda do |sources|
 				sources.map do |source|


@@ 28,13 29,17 @@ class TestResolve < Minitest::Test
 		)
 	end
 
+	def subject(expr)
+		expr.resolve(resolver: @resolver, relative_to: @relative_to).sync
+	end
+
 	def test_nothing_to_resolve
 		expr = Dhall::Function.of_arguments(
 			Dhall::Variable["Natural"],
 			body: Dhall::Variable["_"]
 		)
 
-		assert_equal expr, expr.resolve(Dhall::Resolvers::None.new).sync
+		assert_equal expr, expr.resolve(resolver: Dhall::Resolvers::None.new).sync
 	end
 
 	def test_import_as_text


@@ 44,10 49,7 @@ class TestResolve < Minitest::Test
 			Dhall::Import::RelativePath.new("text")
 		)
 
-		assert_equal(
-			Dhall::Text.new(value: "hai"),
-			expr.resolve(@resolver).sync
-		)
+		assert_equal Dhall::Text.new(value: "hai"), subject(expr)
 	end
 
 	def test_one_level_to_resolve


@@ 60,10 62,7 @@ class TestResolve < Minitest::Test
 			)
 		)
 
-		assert_equal(
-			expr.with(body: Dhall::Variable["_"]),
-			expr.resolve(@resolver).sync
-		)
+		assert_equal expr.with(body: Dhall::Variable["_"]), subject(expr)
 	end
 
 	def test_two_levels_to_resolve


@@ 76,10 75,7 @@ class TestResolve < Minitest::Test
 			)
 		)
 
-		assert_equal(
-			expr.with(body: Dhall::Variable["_"]),
-			expr.resolve(@resolver).sync
-		)
+		assert_equal expr.with(body: Dhall::Variable["_"]), subject(expr)
 	end
 
 	def test_self_loop


@@ 93,7 89,7 @@ class TestResolve < Minitest::Test
 		)
 
 		assert_raises Dhall::ImportLoopException do
-			expr.resolve(@resolver).sync
+			subject(expr)
 		end
 	end
 


@@ 108,7 104,7 @@ class TestResolve < Minitest::Test
 		)
 
 		assert_raises Dhall::ImportLoopException do
-			expr.resolve(@resolver).sync
+			subject(expr)
 		end
 	end
 


@@ 124,7 120,7 @@ class TestResolve < Minitest::Test
 				lhs: Dhall::Text.new(value: "hai"),
 				rhs: Dhall::Text.new(value: "hai")
 			),
-			expr.resolve(@resolver).sync
+			subject(expr)
 		)
 	end
 


@@ 136,7 132,7 @@ class TestResolve < Minitest::Test
 		)
 
 		assert_raises Dhall::ImportFailedException do
-			expr.resolve(@resolver).sync
+			subject(expr)
 		end
 	end
 


@@ 148,7 144,7 @@ class TestResolve < Minitest::Test
 		)
 
 		assert_raises Dhall::Import::IntegrityCheck::FailureException do
-			expr.resolve(@resolver).sync
+			subject(expr)
 		end
 	end
 


@@ 162,7 158,10 @@ class TestResolve < Minitest::Test
 			rhs: Dhall::Variable["fallback"]
 		)
 
-		assert_equal Dhall::Variable["fallback"], expr.resolve(@resolver).sync
+		assert_equal(
+			Dhall::Variable["fallback"],
+			subject(expr)
+		)
 	end
 
 	def test_fallback_to_import


@@ 179,7 178,7 @@ class TestResolve < Minitest::Test
 			)
 		)
 
-		assert_equal Dhall::Variable["_"], expr.resolve(@resolver).sync
+		assert_equal Dhall::Variable["_"], subject(expr)
 	end
 
 	def test_headers


@@ 193,7 192,7 @@ class TestResolve < Minitest::Test
 			Dhall::Import::RelativePath.new("using")
 		)
 
-		assert_equal Dhall::Variable["_"], expr.resolve(@resolver).sync
+		assert_equal Dhall::Variable["_"], subject(expr)
 	end
 
 	def test_ipfs


@@ 226,10 225,43 @@ class TestResolve < Minitest::Test
 	end
 
 	DIRPATH = Pathname.new(File.dirname(__FILE__))
-	TESTS = DIRPATH + "../dhall-lang/tests/normalization/"
+	TESTS = DIRPATH + "../dhall-lang/tests/import/"
+
+	Pathname.glob(TESTS + "success/**/*A.dhall").each do |path|
+		test = path.relative_path_from(TESTS).to_s.sub(/A\.dhall$/, "")
+
+		define_method("test_#{test}") do
+			assert_equal(
+				Dhall::Parser.parse_file(TESTS + "#{test}B.dhall").value,
+				Dhall::Parser.parse_file(path).value.resolve(
+					relative_to: Dhall::Import::Path.from_string(path)
+				).sync
+			)
+		end
+	end
+
+	Pathname.glob(TESTS + "failure/**/*.dhall").each do |path|
+		test = path.relative_path_from(TESTS).to_s.sub(/\.dhall$/, "")
+
+		define_method("test_#{test}") do
+			stub_request(
+				:get,
+				"https://raw.githubusercontent.com/dhall-lang/dhall-lang/" \
+				"master/tests/import/data/referentiallyOpaque.dhall"
+			).to_return(status: 200, body: "env:HOME as Text")
+
+			assert_raises Dhall::ImportFailedException do
+				Dhall::Parser.parse_file(path).value.resolve(
+					relative_to: Dhall::Import::Path.from_string(path)
+				).sync
+			end
+		end
+	end
+
+	NTESTS = DIRPATH + "../dhall-lang/tests/normalization/"
 
 	# Sanity check that all expressions can pass through the resolver
-	Pathname.glob(TESTS + "**/*A.dhall").each do |path|
+	Pathname.glob(NTESTS + "**/*A.dhall").each do |path|
 		test = path.relative_path_from(TESTS).to_s.sub(/A\.dhall$/, "")
 		next if test =~ /prelude\/|remoteSystems/