~singpolyma/dhall-ruby

4c496fb3098b592f24584e64a4cc762d652b745d — Stephen Paul Weber 4 years ago 57b0f3c
Pass all import tests
M lib/dhall.rb => lib/dhall.rb +1 -1
@@ 4,7 4,7 @@ module Dhall
	def self.load_raw(source)
		begin
			return from_binary(source) if source.encoding == Encoding::BINARY
		rescue
		rescue Exception # rubocop:disable Lint/RescueException
			# Parsing CBOR failed, so guess this is source text in standard UTF-8
			return load_raw(source.force_encoding("UTF-8"))
		end

M lib/dhall/ast.rb => lib/dhall/ast.rb +151 -20
@@ 805,6 805,10 @@ module Dhall
			[Double.new(value: other.to_f), self]
		end

		def eql?(other)
			other.is_a?(Double) && to_cbor == other.to_cbor
		end

		def single?
			[value].pack("g").unpack("g").first == value
		end


@@ 898,11 902,12 @@ module Dhall
			end

			def check(expr)
				if @protocol != :nocheck && expr.cache_key != to_s
					raise FailureException, "#{expr} does not match #{self}"
				end
				return expr if @protocol == :nocheck

				expr = expr.normalize
				return expr if expr.cache_key == to_s

				expr
				raise FailureException, "#{expr} does not match #{self}"
			end

			def as_json


@@ 936,10 941,10 @@ module Dhall

			def with(hash)
				self.class.new(
					hash.fetch(:headers),
					authority,
					*path,
					query
					hash.fetch(:headers, headers),
					hash.fetch(:authority, authority),
					*hash.fetch(:path, path),
					hash.fetch(:query, query)
				)
			end



@@ 958,7 963,35 @@ module Dhall
			end

			def uri
				URI("#{scheme}://#{authority}/#{path.join("/")}?#{query}")
				escaped_path = path.map do |c|
					::URI.encode_www_form_component(c).gsub("+", "%20")
				end
				URI("#{scheme}://#{authority}/#{escaped_path.join("/")}?#{query}")
			end

			def chain_onto(relative_to)
				if headers.is_a?(Import)
					with(headers: headers.with(path: headers.real_path(relative_to)))
				else
					self
				end
			end

			def canonical
				with(
					path: (path[1..-1] + [""])
					.reduce([[], path.first]) { |(pth, prev), c|
						c == ".." ? [pth, prev] : [pth + [prev], c]
					}.first.reject { |c| c == "." }
				)
			end

			def origin
				"#{scheme}://#{authority}"
			end

			def to_s
				uri.to_s
			end

			def as_json


@@ 995,8 1028,12 @@ module Dhall
				super(path: path)
			end

			def with(path:)
				self.class.new(*path)
			end

			def self.from_string(s)
				parts = s.split(/\//)
				parts = s.to_s.split(/\//)
				if parts.first == ""
					AbsolutePath.new(*parts[1..-1])
				elsif parts.first == "~"


@@ 1006,10 1043,18 @@ module Dhall
				end
			end

			def canonical
				self.class.from_string(pathname.cleanpath)
			end

			def resolve(resolver)
				resolver.resolve_path(self)
			end

			def origin
				"localhost"
			end

			def to_s
				pathname.to_s
			end


@@ 1027,24 1072,52 @@ module Dhall
			def to_uri(scheme, authority)
				scheme.new(nil, authority, *path, nil)
			end

			def chain_onto(relative_to)
				if relative_to.is_a?(URI)
					raise ImportBannedException, "remote import cannot import #{self}"
				end

				self
			end
		end

		class RelativePath < Path
			def pathname
				Pathname.new(".").join(*path)
			end

			def chain_onto(relative_to)
				relative_to.with(
					path: relative_to.path[0..-2] + path
				)
			end
		end

		class RelativeToParentPath < Path
			def pathname
				Pathname.new("..").join(*path)
			end

			def chain_onto(relative_to)
				relative_to.with(
					path: relative_to.path[0..-2] + [".."] + path
				)
			end
		end

		class RelativeToHomePath < Path
			def pathname
				Pathname.new("~").join(*@path)
			end

			def chain_onto(*)
				if relative_to.is_a?(URI)
					raise ImportBannedException, "remote import cannot import #{self}"
				end

				self
			end
		end

		class EnvironmentVariable


@@ 1070,23 1143,43 @@ module Dhall
				@var = var
			end

			def value
				ENV.fetch(@var)
			def chain_onto(relative_to)
				if relative_to.is_a?(URI)
					raise ImportBannedException, "remote import cannot import #{self}"
				end

				real_path.chain_onto(relative_to)
			end

			def canonical
				real_path.canonical
			end

			def real_path
				val = ENV.fetch(@var) do
					raise ImportFailedException, "No #{self}"
				end
				if val =~ /\Ahttps?:\/\//
					URI.from_uri(URI(val))
				else
					Path.from_string(val)
				end
			end

			def resolve(resolver)
				Promise.resolve(nil).then do
					val = ENV.fetch(@var) do
						raise ImportFailedException, "No ENV #{@var}"
					end
					if val =~ /\Ahttps?:\/\//
						URI.from_uri(URI(value))
					else
						Path.from_string(val)
					end.resolve(resolver)
					real_path.resolve(resolver)
				end
			end

			def origin
				"localhost"
			end

			def to_s
				"env:#{as_json}"
			end

			def as_json
				@var.gsub(/[\"\\\a\b\f\n\r\t\v]/) do |c|
					"\\" + ESCAPES.find { |(_, v)| v == c }.first


@@ 1095,10 1188,24 @@ module Dhall
		end

		class MissingImport
			def chain_onto(*)
				self
			end

			def canonical
				self
			end

			def resolve(*)
				Promise.new.reject(ImportFailedException.new("missing"))
			end

			def origin; end

			def to_s
				"missing"
			end

			def as_json
				[]
			end


@@ 1141,6 1248,30 @@ module Dhall
			)
		end

		def with(options)
			self.class.new(
				options.fetch(:integrity_check, integrity_check),
				options.fetch(:import_type, import_type),
				options.fetch(:path, path)
			)
		end

		def real_path(relative_to)
			path.chain_onto(relative_to).canonical
		end

		def parse_and_check(raw)
			integrity_check.check(import_type.call(raw))
		end

		def cache_key(relative_to)
			if integrity_check.protocol == :nocheck
				real_path(relative_to).to_s
			else
				integrity_check.to_s
			end
		end

		def as_json
			[
				24,

M lib/dhall/normalize.rb => lib/dhall/normalize.rb +1 -1
@@ 18,7 18,7 @@ module Dhall
					x.map(&block)
				end,
				ExpressionHash                => lambda do |x|
					Hash[x.map { |k, v| [k, v.nil? ? v : block[v]] }]
					Hash[x.map { |k, v| [k, v.nil? ? v : block[v]] }.sort]
				end
			)
		end

M lib/dhall/parser.rb => lib/dhall/parser.rb +13 -8
@@ 497,19 497,24 @@ module Dhall
				"https" => Dhall::Import::Https
			}.freeze

			def self.escape(s)
				URI.encode_www_form_component(s).gsub("+", "%20")
			end

			def value
				http = capture(:http_raw)
				SCHEME.fetch(http.capture(:scheme).value).new(
					capture(:import_hashed)&.value(Dhall::Import::Expression),
					http.capture(:authority).value,
					*http.capture(:path).captures(:path_component).map(&:value),
					*unescaped_components,
					http.capture(:query)&.value
				)
			end

			def unescaped_components
				capture(:http_raw)
					.capture(:path)
					.captures(:path_component)
					.map do |pc|
						pc.value(URI.method(:unescape))
					end
			end
		end

		module Env


@@ 575,11 580,11 @@ module Dhall
		end

		module PathComponent
			def value(escaper=:itself.to_proc)
			def value(unescaper=:itself.to_proc)
				if captures.key?(:quoted_path_component)
					escaper.call(capture(:quoted_path_component).value)
					capture(:quoted_path_component).value
				else
					capture(:unquoted_path_component).value
					unescaper.call(capture(:unquoted_path_component).value)
				end
			end
		end

M lib/dhall/resolve.rb => lib/dhall/resolve.rb +99 -31
@@ 1,7 1,8 @@
# frozen_string_literal: true

require "set"
require "pathname"
require "promise.rb"
require "set"

require "dhall/ast"
require "dhall/binary"


@@ 15,13 16,36 @@ module Dhall
	module Resolvers
		ReadPathSources = lambda do |sources|
			sources.map do |source|
				Promise.resolve(nil).then { source.pathname.read }
				Promise.resolve(nil).then { source.pathname.binread }
			end
		end

		ReadHttpSources = lambda do |sources|
		PreflightCORS = lambda do |source, parent_origin|
			uri = source.uri
			if parent_origin != "localhost" && parent_origin != source.origin
				req = Net::HTTP::Options.new(uri)
				req["Origin"] = parent_origin
				req["Access-Control-Request-Method"] = "GET"
				req["Access-Control-Request-Headers"] =
					source.headers.map { |h| h.fetch("header").to_s }.join(",")
				r = Net::HTTP.start(
					uri.hostname,
					uri.port,
					use_ssl: uri.scheme == "https"
				) { |http| http.request(req) }

				raise ImportFailedException, source if r.code != "200"
				unless r["Access-Control-Allow-Origin"] == parent_origin ||
				       r["Access-Control-Allow-Origin"] == "*"
					raise ImportBannedException, source
				end
			end
		end

		ReadHttpSources = lambda do |sources, parent_origin|
			sources.map do |source|
				Promise.resolve(nil).then do
					PreflightCORS.call(source, parent_origin)
					uri = source.uri
					req = Net::HTTP::Get.new(uri)
					source.headers.each do |header|


@@ 58,6 82,10 @@ module Dhall
				@public_gateway = public_gateway
			end

			def arity
				1
			end

			def call(sources)
				@path_reader.call(sources).map.with_index do |promise, idx|
					source = sources[idx]


@@ 80,21 108,19 @@ module Dhall
				promise.catch {
					@http_reader.call([
						source.to_uri(Import::Http, "localhost:8000")
					]).first
					], "localhost").first
				}.catch do
					@https_reader.call([
						source.to_uri(Import::Https, @public_gateway)
					]).first
					], "localhost").first
				end
			end
		end

		class ResolutionSet
			attr_reader :reader

			def initialize(reader)
				@reader = reader
				@parents = Set.new
				@parents = []
				@set = Hash.new { |h, k| h[k] = [] }
			end



@@ 113,6 139,16 @@ module Dhall
				[Array(sources), Array(promises)]
			end

			def reader
				lambda do |sources|
					if @reader.arity == 2
						@reader.call(sources, @parents.last&.origin || "localhost")
					else
						@reader.call(sources)
					end
				end
			end

			def child(parent_source)
				dup.tap do |c|
					c.instance_eval do


@@ 132,6 168,15 @@ module Dhall
				@path_resolutions = ResolutionSet.new(path_reader)
				@http_resolutions = ResolutionSet.new(http_reader)
				@https_resolutions = ResolutionSet.new(https_reader)
				@cache = {}
			end

			def cache_fetch(key, &fallback)
				@cache.fetch(key) do
					Promise.resolve(nil).then(&fallback).then do |result|
						@cache[key] = result
					end
				end
			end

			def resolve_path(path_source)


@@ 139,9 184,10 @@ module Dhall
			end

			def resolve_http(http_source)
				ExpressionResolver
					.for(http_source.headers)
					.resolve(self).then do |headers|
				http_source.headers.resolve(
					resolver:    self,
					relative_to: Dhall::Import::RelativePath.new
				).then do |headers|
					@http_resolutions.register(
						http_source.with(headers: headers.normalize)
					)


@@ 149,9 195,10 @@ module Dhall
			end

			def resolve_https(https_source)
				ExpressionResolver
					.for(https_source.headers)
					.resolve(self).then do |headers|
				https_source.headers.resolve(
					resolver:    self,
					relative_to: Dhall::Import::RelativePath.new
				).then do |headers|
					@https_resolutions.register(
						https_source.with(headers: headers.normalize)
					)


@@ 236,10 283,10 @@ module Dhall
			@expr = expr
		end

		def resolve(resolver)
		def resolve(**kwargs)
			Util.promise_all_hash(
				@expr.to_h.each_with_object({}) { |(attr, value), h|
					h[attr] = ExpressionResolver.for(value).resolve(resolver)
					h[attr] = ExpressionResolver.for(value).resolve(**kwargs)
				}
			).then { |h| @expr.with(h) }
		end


@@ 247,39 294,54 @@ module Dhall
		class ImportResolver < ExpressionResolver
			register_for Import

			def resolve(resolver)
				@expr.instance_eval do
					@path.resolve(resolver).then do |result|
						@integrity_check.check(
							@import_type.call(result)
						).resolve(resolver.child(@path))
			def resolve(resolver:, relative_to:)
				Promise.resolve(nil).then do
					resolver.cache_fetch(@expr.cache_key(relative_to)) do
						resolve_raw(resolver: resolver, relative_to: relative_to)
					end
				end
			end

			def resolve_raw(resolver:, relative_to:)
				real_path = @expr.real_path(relative_to)
				real_path.resolve(resolver).then do |result|
					@expr.parse_and_check(result).resolve(
						resolver:    resolver.child(real_path),
						relative_to: real_path
					)
				end
			end
		end

		class FallbackResolver < ExpressionResolver
			register_for Operator::ImportFallback

			def resolve(resolver)
				ExpressionResolver.for(@expr.lhs).resolve(resolver).catch do
					ExpressionResolver.for(@expr.rhs).resolve(resolver)
			def resolve(**kwargs)
				ExpressionResolver.for(@expr.lhs).resolve(**kwargs).catch do
					ExpressionResolver.for(@expr.rhs).resolve(**kwargs)
				end
			end
		end

		class ArrayResolver < ExpressionResolver
			def resolve(resolver)
			register_for Util::ArrayOf.new(Expression)

			def resolve(**kwargs)
				Promise.all(
					@expr.map { |e| ExpressionResolver.for(e).resolve(resolver) }
					@expr.map { |e| ExpressionResolver.for(e).resolve(**kwargs) }
				)
			end
		end

		class HashResolver < ExpressionResolver
			def resolve(resolver)
			register_for Util::HashOf.new(
				ValueSemantics::Anything,
				ValueSemantics::Either.new([Expression, nil])
			)

			def resolve(**kwargs)
				Util.promise_all_hash(Hash[@expr.map do |k, v|
					[k, ExpressionResolver.for(v).resolve(resolver)]
					[k, ExpressionResolver.for(v).resolve(**kwargs)]
				end])
			end
		end


@@ 296,8 358,14 @@ module Dhall
	end

	class Expression
		def resolve(resolver=Resolvers::Default.new)
			p = ExpressionResolver.for(self).resolve(resolver)
		def resolve(
			resolver: Resolvers::Default.new,
			relative_to: Import::Path.from_string(Pathname.pwd + "file")
		)
			p = ExpressionResolver.for(self).resolve(
				resolver:    resolver,
				relative_to: relative_to
			)
			resolver.finish!
			p
		end

M test/test_resolve.rb => test/test_resolve.rb +55 -23
@@ 8,6 8,7 @@ require "dhall"

class TestResolve < Minitest::Test
	def setup
		@relative_to = Dhall::Import::RelativePath.new
		@resolver = Dhall::Resolvers::Default.new(
			path_reader: lambda do |sources|
				sources.map do |source|


@@ 28,13 29,17 @@ class TestResolve < Minitest::Test
		)
	end

	def subject(expr)
		expr.resolve(resolver: @resolver, relative_to: @relative_to).sync
	end

	def test_nothing_to_resolve
		expr = Dhall::Function.of_arguments(
			Dhall::Variable["Natural"],
			body: Dhall::Variable["_"]
		)

		assert_equal expr, expr.resolve(Dhall::Resolvers::None.new).sync
		assert_equal expr, expr.resolve(resolver: Dhall::Resolvers::None.new).sync
	end

	def test_import_as_text


@@ 44,10 49,7 @@ class TestResolve < Minitest::Test
			Dhall::Import::RelativePath.new("text")
		)

		assert_equal(
			Dhall::Text.new(value: "hai"),
			expr.resolve(@resolver).sync
		)
		assert_equal Dhall::Text.new(value: "hai"), subject(expr)
	end

	def test_one_level_to_resolve


@@ 60,10 62,7 @@ class TestResolve < Minitest::Test
			)
		)

		assert_equal(
			expr.with(body: Dhall::Variable["_"]),
			expr.resolve(@resolver).sync
		)
		assert_equal expr.with(body: Dhall::Variable["_"]), subject(expr)
	end

	def test_two_levels_to_resolve


@@ 76,10 75,7 @@ class TestResolve < Minitest::Test
			)
		)

		assert_equal(
			expr.with(body: Dhall::Variable["_"]),
			expr.resolve(@resolver).sync
		)
		assert_equal expr.with(body: Dhall::Variable["_"]), subject(expr)
	end

	def test_self_loop


@@ 93,7 89,7 @@ class TestResolve < Minitest::Test
		)

		assert_raises Dhall::ImportLoopException do
			expr.resolve(@resolver).sync
			subject(expr)
		end
	end



@@ 108,7 104,7 @@ class TestResolve < Minitest::Test
		)

		assert_raises Dhall::ImportLoopException do
			expr.resolve(@resolver).sync
			subject(expr)
		end
	end



@@ 124,7 120,7 @@ class TestResolve < Minitest::Test
				lhs: Dhall::Text.new(value: "hai"),
				rhs: Dhall::Text.new(value: "hai")
			),
			expr.resolve(@resolver).sync
			subject(expr)
		)
	end



@@ 136,7 132,7 @@ class TestResolve < Minitest::Test
		)

		assert_raises Dhall::ImportFailedException do
			expr.resolve(@resolver).sync
			subject(expr)
		end
	end



@@ 148,7 144,7 @@ class TestResolve < Minitest::Test
		)

		assert_raises Dhall::Import::IntegrityCheck::FailureException do
			expr.resolve(@resolver).sync
			subject(expr)
		end
	end



@@ 162,7 158,10 @@ class TestResolve < Minitest::Test
			rhs: Dhall::Variable["fallback"]
		)

		assert_equal Dhall::Variable["fallback"], expr.resolve(@resolver).sync
		assert_equal(
			Dhall::Variable["fallback"],
			subject(expr)
		)
	end

	def test_fallback_to_import


@@ 179,7 178,7 @@ class TestResolve < Minitest::Test
			)
		)

		assert_equal Dhall::Variable["_"], expr.resolve(@resolver).sync
		assert_equal Dhall::Variable["_"], subject(expr)
	end

	def test_headers


@@ 193,7 192,7 @@ class TestResolve < Minitest::Test
			Dhall::Import::RelativePath.new("using")
		)

		assert_equal Dhall::Variable["_"], expr.resolve(@resolver).sync
		assert_equal Dhall::Variable["_"], subject(expr)
	end

	def test_ipfs


@@ 226,10 225,43 @@ class TestResolve < Minitest::Test
	end

	DIRPATH = Pathname.new(File.dirname(__FILE__))
	TESTS = DIRPATH + "../dhall-lang/tests/normalization/"
	TESTS = DIRPATH + "../dhall-lang/tests/import/"

	Pathname.glob(TESTS + "success/**/*A.dhall").each do |path|
		test = path.relative_path_from(TESTS).to_s.sub(/A\.dhall$/, "")

		define_method("test_#{test}") do
			assert_equal(
				Dhall::Parser.parse_file(TESTS + "#{test}B.dhall").value,
				Dhall::Parser.parse_file(path).value.resolve(
					relative_to: Dhall::Import::Path.from_string(path)
				).sync
			)
		end
	end

	Pathname.glob(TESTS + "failure/**/*.dhall").each do |path|
		test = path.relative_path_from(TESTS).to_s.sub(/\.dhall$/, "")

		define_method("test_#{test}") do
			stub_request(
				:get,
				"https://raw.githubusercontent.com/dhall-lang/dhall-lang/" \
				"master/tests/import/data/referentiallyOpaque.dhall"
			).to_return(status: 200, body: "env:HOME as Text")

			assert_raises Dhall::ImportFailedException do
				Dhall::Parser.parse_file(path).value.resolve(
					relative_to: Dhall::Import::Path.from_string(path)
				).sync
			end
		end
	end

	NTESTS = DIRPATH + "../dhall-lang/tests/normalization/"

	# Sanity check that all expressions can pass through the resolver
	Pathname.glob(TESTS + "**/*A.dhall").each do |path|
	Pathname.glob(NTESTS + "**/*A.dhall").each do |path|
		test = path.relative_path_from(TESTS).to_s.sub(/A\.dhall$/, "")
		next if test =~ /prelude\/|remoteSystems/