From 3dae10e1594374c8a64de89ce83278a1822e7758 Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber Date: Wed, 10 Jul 2019 22:00:54 -0500 Subject: [PATCH] implement as Location and use URI internally --- dhall.gemspec | 1 + lib/dhall/ast.rb | 132 ++++++++++++++++++++++++----------------- lib/dhall/binary.rb | 36 ++++++++++- lib/dhall/parser.rb | 65 ++++++++++++++------ lib/dhall/resolve.rb | 16 +++++ lib/dhall/typecheck.rb | 2 +- lib/dhall/util.rb | 8 +++ test/test_resolve.rb | 44 ++++++++++++-- test/test_resolvers.rb | 8 +-- 9 files changed, 229 insertions(+), 83 deletions(-) diff --git a/dhall.gemspec b/dhall.gemspec index 69912c6..de5183f 100644 --- a/dhall.gemspec +++ b/dhall.gemspec @@ -27,6 +27,7 @@ Gem::Specification.new do |spec| spec.add_dependency "cbor", "~> 0.5.9.3" spec.add_dependency "citrus", "~> 3.0" + spec.add_dependency "lazy_object", "~> 0.0.3" spec.add_dependency "multihashes", "~> 0.1.3" spec.add_dependency "promise.rb", "~> 0.7.4" spec.add_dependency "value_semantics", "~> 3.0" diff --git a/lib/dhall/ast.rb b/lib/dhall/ast.rb index 0d934e8..1c5cec7 100644 --- a/lib/dhall/ast.rb +++ b/lib/dhall/ast.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require "lazy_object" require "multihashes" require "uri" require "value_semantics" @@ -1256,60 +1257,43 @@ module Dhall end end + Location = LazyObject.new do + UnionType.new( + alternatives: { + "Local" => Builtins[:Text], + "Remote" => Builtins[:Text], + "Environment" => Builtins[:Text], + "Missing" => nil + } + ) + end + class URI include(ValueSemantics.for_attributes do - headers Either(nil, Expression) - authority ::String - path ArrayOf(::String) - query Either(nil, ::String) + uri ::URI + headers Either(nil, Expression), default: nil end) - def initialize(headers, authority, *path, query) - super( - headers: headers, - authority: authority, - path: path, - query: query, - ) - end - - def with(hash) - self.class.new( - hash.fetch(:headers, headers), - hash.fetch(:authority, authority), - *hash.fetch(:path, path), - hash.fetch(:query, query) - ) - end + def with(attrs) + if attrs.key?(:path) + attrs[:uri] = + uri + Util.path_components_to_uri(*attrs.delete(:path)) + end - def self.from_uri(uri) - (uri.scheme == "https" ? Https : Http).new( - nil, - "#{uri.host}:#{uri.port}", - *uri.path.split(/\//)[1..-1], - uri.query, - nil - ) + super end def headers header_type = RecordType.new( record: { - "header" => Builtins[:Text], - "value" => Builtins[:Text] + "mapKey" => Builtins[:Text], + "mapValue" => Builtins[:Text] } ) super || EmptyList.new(element_type: header_type) end - def uri - escaped_path = path.map do |c| - ::URI.encode_www_form_component(c).gsub("+", "%20") - end - URI("#{scheme}://#{authority}/#{escaped_path.join("/")}?#{query}") - end - def chain_onto(relative_to) if headers.is_a?(Import) with(headers: headers.with(path: headers.real_path(relative_to))) @@ -1320,23 +1304,43 @@ module Dhall def canonical with( - path: (path[1..-1] + [""]) - .reduce([[], path.first]) { |(pth, prev), c| + path: (path[1..-1] + [""]).reduce([[], path.first]) { |(pth, prev), c| c == ".." ? [pth, prev] : [pth + [prev], c] }.first.reject { |c| c == "." } ) end + def port + uri.port && uri.port != uri.default_port ? uri.port : nil + end + + def authority + [ + uri.userinfo, + [uri.host, port].compact.join(":") + ].compact.join("@") + end + def origin - "#{scheme}://#{authority}" + "#{uri.scheme}://#{authority}" end def to_s uri.to_s end + def location + Union.from(Location, "Remote", to_s.as_dhall) + end + + def path + path = uri.path.split(/\//, -1).map(&::URI.method(:unescape)) + path = path[1..-1] if path.length > 1 && path.first.empty? + path + end + def as_json - [@headers&.as_json, authority, *path, query] + [@headers&.as_json, authority, *path, uri.query] end end @@ -1344,20 +1348,12 @@ module Dhall def resolve(resolver) resolver.resolve_http(self) end - - def scheme - "http" - end end class Https < URI def resolve(resolver) resolver.resolve_https(self) end - - def scheme - "https" - end end class Path @@ -1402,6 +1398,10 @@ module Dhall pathname.to_s end + def location + Union.from(Location, "Local", to_s.as_dhall) + end + def as_json path end @@ -1412,8 +1412,8 @@ module Dhall Pathname.new("/").join(*path) end - def to_uri(scheme, authority) - scheme.new(nil, authority, *path, nil) + def to_uri(scheme, base_uri) + scheme.new(uri: base_uri + Util.path_components_to_uri(*path)) end def chain_onto(relative_to) @@ -1430,6 +1430,10 @@ module Dhall Pathname.new(".").join(*path) end + def to_s + "./#{pathname}" + end + def chain_onto(relative_to) relative_to.with( path: relative_to.path[0..-2] + path @@ -1509,6 +1513,10 @@ module Dhall end}" end + def location + Union.from(Location, "Environment", to_s.as_dhall) + end + def hash @var.hash end @@ -1516,7 +1524,7 @@ module Dhall def eql?(other) other.is_a?(self.class) && other.var == var end - alias eql? == + alias == eql? def as_json @var @@ -1542,6 +1550,15 @@ module Dhall "missing" end + def location + Union.from(Location, "Missing", nil) + end + + def eql?(other) + other.class == self.class + end + alias == eql? + def as_json [] end @@ -1561,9 +1578,16 @@ module Dhall end end + class AsLocation + def self.call(*) + raise "AsLocation is only a marker, you don't actually call it" + end + end + IMPORT_TYPES = [ Expression, - Text + Text, + AsLocation ].freeze PATH_TYPES = [ diff --git a/lib/dhall/binary.rb b/lib/dhall/binary.rb index 0c62aab..9bdfb85 100644 --- a/lib/dhall/binary.rb +++ b/lib/dhall/binary.rb @@ -6,6 +6,7 @@ require "multihashes" require "dhall/ast" require "dhall/builtins" +require "dhall/parser" module Dhall def self.from_binary(cbor_binary) @@ -210,13 +211,46 @@ module Dhall end end + class URI + def self.decode(headers, authority, *path, query) + new( + headers: headers, + uri: ::URI.scheme_list[name.split(/::/).last.upcase].build( + Parser.parse(authority, root: :authority).value.merge( + path: Util.path_components_to_uri(*path).path, + query: query + ) + ) + ) + end + end + + class Path + def self.decode(*args) + new(*args) + end + end + + class EnvironmentVariable + def self.decode(*args) + new(*args) + end + end + + class MissingImport + def self.decode(*args) + new(*args) + end + end + def self.decode(integrity_check, import_type, path_type, *parts) parts[0] = Dhall.decode(parts[0]) if path_type < 2 && !parts[0].nil? + path_type = PATH_TYPES.fetch(path_type) new( IntegrityCheck.decode(integrity_check), IMPORT_TYPES[import_type], - PATH_TYPES[path_type].new(*parts) + path_type.decode(*parts) ) end end diff --git a/lib/dhall/parser.rb b/lib/dhall/parser.rb index 3013392..1913628 100644 --- a/lib/dhall/parser.rb +++ b/lib/dhall/parser.rb @@ -510,6 +510,8 @@ module Dhall def value import_type = if captures.key?(:text) Dhall::Import::Text + elsif captures.key?(:location) + Dhall::Import::AsLocation else Dhall::Import::Expression end @@ -538,29 +540,45 @@ module Dhall end end + module Scheme + def value + ::URI.scheme_list[string.upcase] + end + end + + module Authority + def value + { + userinfo: capture(:userinfo)&.value, + host: capture(:host).value, + port: capture(:port)&.value + } + end + end + module Http SCHEME = { "http" => Dhall::Import::Http, "https" => Dhall::Import::Https }.freeze + def http(key) + @http ||= capture(:http_raw) + @http.capture(key)&.value + end + def value - http = capture(:http_raw) - SCHEME.fetch(http.capture(:scheme).value).new( - capture(:import_hashed)&.value(Dhall::Import::Expression), - http.capture(:authority).value, - *unescaped_components, - http.capture(:query)&.value + uri = http(:scheme).build( + http(:authority).merge( + path: http(:url_path) || "/", + query: http(:query) + ) ) - end - def unescaped_components - capture(:http_raw) - .capture(:path) - .captures(:path_component) - .map do |pc| - pc.value(URI.method(:unescape)) - end + SCHEME.fetch(uri.scheme).new( + headers: capture(:import_expression)&.value, + uri: uri + ) end end @@ -637,15 +655,28 @@ module Dhall end module PathComponent - def value(unescaper=:itself.to_proc) + def value(escaper=:itself.to_proc) if captures.key?(:quoted_path_component) - capture(:quoted_path_component).value + escaper.call(capture(:quoted_path_component).value) else - unescaper.call(capture(:unquoted_path_component).value) + capture(:unquoted_path_component).value end end end + module UrlPath + def value + "/" + matches.map { |pc| + if pc.captures.key?(:path_component) + # We escape here because ruby stdlib URI just stores path unparsed + pc.value(Util.method(:uri_escape)) + else + pc.string[1..-1] + end + }.join("/") + end + end + module Missing def value Dhall::Import::MissingImport.new diff --git a/lib/dhall/resolve.rb b/lib/dhall/resolve.rb index d98278c..57006f9 100644 --- a/lib/dhall/resolve.rb +++ b/lib/dhall/resolve.rb @@ -427,9 +427,25 @@ module Dhall ).then { |h| @expr.with(h) } end + class ImportAsLocationResolver < ExpressionResolver + def resolve(resolver:, relative_to:) + Promise.resolve(nil).then do + @expr.real_path(relative_to).location + end + end + end + class ImportResolver < ExpressionResolver register_for Import + def self.new(expr) + if expr.import_type == Import::AsLocation + ImportAsLocationResolver.new(expr) + else + super + end + end + def resolve(resolver:, relative_to:) Promise.resolve(nil).then do resolver.cache_fetch(@expr.cache_key(relative_to)) do diff --git a/lib/dhall/typecheck.rb b/lib/dhall/typecheck.rb index 21be8be..254c8e1 100644 --- a/lib/dhall/typecheck.rb +++ b/lib/dhall/typecheck.rb @@ -12,7 +12,7 @@ module Dhall def self.assert_type(expr, assertion, message, context:) aexpr = self.for(expr).annotate(context) - type = aexpr.type + type = aexpr.type.normalize raise TypeError, "#{message}: #{type}" unless assertion === type aexpr end diff --git a/lib/dhall/util.rb b/lib/dhall/util.rb index 600fd35..09280e1 100644 --- a/lib/dhall/util.rb +++ b/lib/dhall/util.rb @@ -192,5 +192,13 @@ module Dhall .reduce(&method(:longest_common_prefix))&.length.to_i end end + + def self.path_components_to_uri(*components) + URI("/#{components.map(&method(:uri_escape)).join("/")}") + end + + def self.uri_escape(s) + ::URI.encode_www_form_component(s).gsub("+", "%20") + end end end diff --git a/test/test_resolve.rb b/test/test_resolve.rb index b270932..7288863 100644 --- a/test/test_resolve.rb +++ b/test/test_resolve.rb @@ -264,7 +264,7 @@ class TestResolve < Minitest::Test digest: Dhall::Variable["_"].digest.digest ), Dhall::Import::Expression, - Dhall::Import::Http.new(nil, "example.com", "thing.dhall", nil) + Dhall::Import::Http.new(uri: URI("http://example.com/thing.dhall")) ) cache = Dhall::Resolvers::RamCache.new @@ -315,19 +315,51 @@ class TestResolve < Minitest::Test assert_equal Dhall::Variable["_"], expr.resolve.sync end - DIRPATH = Pathname.new(File.dirname(__FILE__)) - TESTS = DIRPATH + "../dhall-lang/tests/import/" + DIRPATH = Pathname.new(File.dirname(__FILE__)).realpath + TESTS = (DIRPATH + "../dhall-lang/tests/import/").realpath Pathname.glob(TESTS + "success/**/*A.dhall").each do |path| + Dir.chdir(path.dirname) + ENV["XDG_CACHE_HOME"] = (TESTS + "cache").to_s + test = path.relative_path_from(TESTS).to_s.sub(/A\.dhall$/, "") define_method("test_#{test}") do + stub_request(:any, "https://httpbin.org/user-agent") + .to_return do |req| + { + headers: { "Access-Control-Allow-Origin": "*" }, + body: <<~JSON + { + "user-agent": "#{req.headers["User-Agent"]}" + } + JSON + } + end + + stub_request( + :get, + "https://raw.githubusercontent.com/dhall-lang/dhall-lang/" \ + "master/tests/import/success/customHeadersA.dhall" + ).to_return(body: (TESTS + "success/customHeadersA.dhall").read) + + stub_request(:get, "https://test.dhall-lang.org/Bool/package.dhall") + .with(headers: { "Test" => "Example" }) + .to_return(body: "./test.dhall") + + stub_request(:get, "https://test.dhall-lang.org/Bool/test.dhall") + .with(headers: { "Test" => "Example" }) + .to_return(body: (TESTS + "#{test}B.dhall").read) + + Dhall::Function.disable_alpha_normalization! assert_equal( - Dhall::Parser.parse_file(TESTS + "#{test}B.dhall").value, + Dhall::Parser.parse_file(TESTS + "#{test}B.dhall").value.normalize, Dhall::Parser.parse_file(path).value.resolve( - relative_to: Dhall::Import::Path.from_string(path) - ).sync + resolver: Dhall::Resolvers::Standard.new, + relative_to: Dhall::Import::Path.from_string("./") + ).sync.normalize ) + Dhall::Function.enable_alpha_normalization! end end diff --git a/test/test_resolvers.rb b/test/test_resolvers.rb index 768087f..28004d9 100644 --- a/test/test_resolvers.rb +++ b/test/test_resolvers.rb @@ -36,7 +36,7 @@ class TestResolvers < Minitest::Test sources.map { |source| Promise.resolve(source.to_s) } end ) - source = Dhall::Import::Http.new(nil, "example.com", "x.dhall", nil) + source = Dhall::Import::Http.new(uri: URI("http://example.com/x.dhall")) promise = source.resolve(resolver) resolver.finish! assert_equal source.to_s, promise.sync @@ -48,7 +48,7 @@ class TestResolvers < Minitest::Test sources.map { |source| Promise.resolve(source.to_s) } end ) - source = Dhall::Import::Https.new(nil, "example.com", "x.dhall", nil) + source = Dhall::Import::Https.new(uri: URI("https://example.com/x.dhall")) promise = source.resolve(resolver) resolver.finish! assert_equal source.to_s, promise.sync @@ -60,7 +60,7 @@ class TestResolvers < Minitest::Test sources.map { |source| Promise.resolve(source.to_s) } end ) - source = Dhall::Import::Https.new(nil, "example.com", "x.dhall", nil) + source = Dhall::Import::Https.new(uri: URI("https://example.com/x.dhall")) promise = source.resolve(resolver) resolver.finish! assert_equal source.to_s, promise.sync @@ -68,7 +68,7 @@ class TestResolvers < Minitest::Test def test_local_only_resolver_rejects_http resolver = Dhall::Resolvers::LocalOnly.new - source = Dhall::Import::Http.new(nil, "example.com", "x.dhall", nil) + source = Dhall::Import::Http.new(uri: URI("http://example.com/x.dhall")) promise = source.resolve(resolver) resolver.finish! assert_raises Dhall::ImportBannedException do -- 2.38.5