diff --git a/.gitignore b/.gitignore index c81aaec..abc1803 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,8 @@ Session.vim coverage .rvmrc *.gem +Gemfile.lock +spec/fixtures/cassettes + +.DS_Store +.ruby-version \ No newline at end of file diff --git a/Gemfile.lock b/Gemfile.lock deleted file mode 100644 index e9193e6..0000000 --- a/Gemfile.lock +++ /dev/null @@ -1,71 +0,0 @@ -PATH - remote: . - specs: - gtfs (0.5.1) - multi_json - rake - rubyzip (~> 2.3) - -GEM - remote: https://rubygems.org/ - specs: - addressable (2.8.7) - public_suffix (>= 2.0.2, < 7.0) - bigdecimal (3.1.8) - byebug (11.1.3) - crack (1.0.0) - bigdecimal - rexml - diff-lcs (1.5.1) - docile (1.4.1) - fakeweb (1.3.0) - hashdiff (1.1.1) - multi_json (1.15.0) - public_suffix (6.0.1) - rake (13.2.1) - rexml (3.3.9) - rspec (3.13.0) - rspec-core (~> 3.13.0) - rspec-expectations (~> 3.13.0) - rspec-mocks (~> 3.13.0) - rspec-core (3.13.0) - rspec-support (~> 3.13.0) - rspec-expectations (3.13.1) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.13.0) - rspec-its (1.3.0) - rspec-core (>= 3.0.0) - rspec-expectations (>= 3.0.0) - rspec-mocks (3.13.1) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.13.0) - rspec-support (3.13.1) - rubyzip (2.3.2) - simplecov (0.22.0) - docile (~> 1.1) - simplecov-html (~> 0.11) - simplecov_json_formatter (~> 0.1) - simplecov-html (0.12.3) - simplecov_json_formatter (0.1.4) - vcr (6.2.0) - webmock (3.23.1) - addressable (>= 2.8.0) - crack (>= 0.3.2) - hashdiff (>= 0.4.0, < 2.0.0) - -PLATFORMS - ruby - -DEPENDENCIES - byebug - fakeweb - gtfs! - rake - rspec - rspec-its - simplecov - vcr - webmock - -BUNDLED WITH - 2.3.6 diff --git a/gtfs.gemspec b/gtfs.gemspec index c829f0d..de38b9c 100644 --- a/gtfs.gemspec +++ b/gtfs.gemspec @@ -21,7 +21,8 @@ Gem::Specification.new do |gem| gem.add_dependency 'rake' gem.add_dependency 'multi_json' gem.add_dependency 'rubyzip', ['~> 2.3'] - + gem.add_dependency 'csv' + gem.add_development_dependency 'rspec', ['>= 2.0.0'] gem.add_development_dependency 'simplecov' gem.add_development_dependency 'byebug' diff --git a/lib/gtfs/url_source.rb b/lib/gtfs/url_source.rb index 0e7e4e1..d4e8750 100644 --- a/lib/gtfs/url_source.rb +++ b/lib/gtfs/url_source.rb @@ -4,11 +4,26 @@ module GTFS class URLSource < Source + MAXIMUM_REDIRECTS = 5 + def load_archive(source_url) + original_source_url = source_url.dup Dir.mktmpdir do |tmp| + n = 0 + while n < MAXIMUM_REDIRECTS + uri = URI.parse(source_url) + response = Net::HTTP.get_response(uri) + if response.is_a?(Net::HTTPRedirection) + n += 1 + source_url = response['location'] + else + break + end + end + if n >= MAXIMUM_REDIRECTS + raise InvalidSourceException.new("Too many redirects for #{original_source_url}") + end file_name = File.join(tmp, "/gtfs_temp_#{Time.now.strftime('%Y%jT%H%M%S%z')}.zip") - uri = URI.parse(source_url) - response = Net::HTTP.get_response(uri) open(file_name, 'wb') do |file| file.write response.body end diff --git a/spec/fixtures/cassettes/invalid_gtfs_uri.yml b/spec/fixtures/cassettes/invalid_gtfs_uri.yml deleted file mode 100644 index cb2e4da..0000000 --- a/spec/fixtures/cassettes/invalid_gtfs_uri.yml +++ /dev/null @@ -1,63 +0,0 @@ ---- -http_interactions: -- request: - method: get - uri: https://example.org/gtfs.zip - body: - encoding: US-ASCII - string: '' - headers: - Accept-Encoding: - - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 - Accept: - - "*/*" - User-Agent: - - Ruby - Host: - - example.org - response: - status: - code: 404 - message: Not Found - headers: - Age: - - '164717' - Cache-Control: - - max-age=604800 - Content-Type: - - text/html; charset=UTF-8 - Date: - - Fri, 23 Oct 2020 23:02:49 GMT - Expires: - - Fri, 30 Oct 2020 23:02:49 GMT - Last-Modified: - - Thu, 22 Oct 2020 01:17:32 GMT - Server: - - ECS (oxr/837A) - Vary: - - Accept-Encoding - X-Cache: - - 404-HIT - Content-Length: - - '648' - body: - encoding: ASCII-8BIT - string: "\n\n\n Example Domain\n\n - \ \n \n \n \n\n\n\n
\n

Example Domain

\n - \

This domain is for use in illustrative examples in documents. You may - use this\n domain in literature without prior coordination or asking for - permission.

\n

More - information...

\n
\n\n\n" - recorded_at: Fri, 23 Oct 2020 23:02:49 GMT -recorded_with: VCR 6.0.0 diff --git a/spec/fixtures/cassettes/valid_gtfs_uri.yml b/spec/fixtures/cassettes/valid_gtfs_uri.yml deleted file mode 100644 index ecb2d19..0000000 --- a/spec/fixtures/cassettes/valid_gtfs_uri.yml +++ /dev/null @@ -1,60 +0,0 @@ ---- -http_interactions: -- request: - method: get - uri: https://developers.google.com/transit/gtfs/examples/sample-feed.zip - body: - encoding: US-ASCII - string: '' - headers: - Accept-Encoding: - - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 - Accept: - - "*/*" - User-Agent: - - Ruby - Host: - - developers.google.com - response: - status: - code: 200 - message: OK - headers: - Content-Type: - - application/zip - Last-Modified: - - Wed, 11 Jan 2012 14:31:33 GMT - Content-Disposition: - - attachment; filename="sample-feed.zip" - Cache-Control: - - no-cache, must-revalidate - Expires: - - '0' - Pragma: - - no-cache - Strict-Transport-Security: - - max-age=31536000; includeSubdomains - X-Frame-Options: - - SAMEORIGIN - X-Xss-Protection: - - '0' - X-Content-Type-Options: - - nosniff - X-Cloud-Trace-Context: - - 9d9cd729a1af3deba482f46f3a68c263 - Date: - - Fri, 23 Oct 2020 23:02:49 GMT - Server: - - Google Frontend - Content-Length: - - '3217' - Alt-Svc: - - h3-Q050=":443"; ma=2592000,h3-29=":443"; ma=2592000,h3-T051=":443"; ma=2592000,h3-T050=":443"; - ma=2592000,h3-Q046=":443"; ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; - ma=2592000; v="46,43" - body: - encoding: ASCII-8BIT - string: !binary |- - UEsDBBQAAAAIACZfiDaYbOk2XQAAAHEAAAAKABUAYWdlbmN5LnR4dFVUCQADBzsZRkk/GUZVeAQA9QEUADXGsQqFMAwF0P19hR9Q7O5WcHR0l1AvNdAk0sZBv97h4ZkOFWi+N94D/ack+H61+tVZ8JjiN68pzBAb1kba2Yd0+WGN/Q6H+znFWMxKxZhNQhI0zhQX61vSgor+AlBLAwQUAAAACADQYIg2PApMwWUAAACiAAAADAAVAGNhbGVuZGFyLnR4dFVUCQADNz0ZRkk/GUZVeAQA9QEUACtOLSrLTE6Nz0zRyc3PS0ms1CkpTS0G0eWpKXkQVklGaRGYkVaUCaKKE0tKi8CMUrCO4pLEopL4lMSSVJ3UvBQwg5fLLdTHJ1zHEAUaGRiYGxgagBhA0sjYkJcr3FXHAA6xqgEAUEsDBAoAAAAAAA1hiDYjfNesMAAAADAAAAASABUAY2FsZW5kYXJfZGF0ZXMudHh0VVQJAAOqPRlGTj8ZRlV4BAD1ARQAc2VydmljZV9pZCxkYXRlLGV4Y2VwdGlvbl90eXBlDQpGVUxMVywyMDA3MDYwNCwyUEsDBBQAAAAIAEFhiDZpEYREVgAAAGgAAAATABUAZmFyZV9hdHRyaWJ1dGVzLnR4dFVUCQADCT4ZRk4/GUZVeAQA9QEUAE3IMQrAIAxA0V3wJqG0BW/QG5TOEjRSB6PEOHj7dirlL4+fUMjnCE1yIAhDhDhMr7MRNJyFWH0hvWsEFeSeSPonH4eg5srWNNiW3cF1HrC+WYPg/uMBUEsDBBQAAAAIAIZhiDb1AII+SQAAAF4AAAAOABUAZmFyZV9ydWxlcy50eHRVVAkAA4w+GUZPPxlGVXgEAPUBFAAtxzEKgDAMAMBd8Cf5RCK4OSmuEmyVLIm08f+2we3u4pIPSVDs9YAVuUW7Uq4uyi4WPU2dRWvzODyABAAd60b4k+YpxIC47I0fUEsDBBQAAAAIALlhiDY6cWBZdwAAAFkBAAAPABUAZnJlcXVlbmNpZXMudHh0VVQJAAPtPhlGUD8ZRlV4BAD1ARQAKynKLIjPTNEpLkksKokvycxN1UnNS4EwMlITU8oTK+OLU5OLuYJDnBx1zKwMDIBIx8gIyjC0MDDgcvYMiTSEy5lbmVoCEULKCLeUoY4FVMoSKmUG14RLxlDH0ABmuymmXXgkgTrN4O7GsA+3HFCfJU6PG+GQBABQSwMEFAAAAAgAMmCINgOVqdu8AAAAOwEAAAoAFQByb3V0ZXMudHh0VVQJAAMQPBlGUD8ZRlV4BAD1ARQATY9BCoMwEEX3gnfIAaZga3uAaBG66KaK0JWEOFVpNDImUG/fNDXg7s+b95mEtDXYDC2IDie5/hJ5tPSaTDOJETeg9NTt5xYXuUWzzoFaUluSWmkKAn7MH8QRz+BacTgmwAea3RF2YJlV6kW6A0gBII6yIvfSKYGwclZhaRISWU6Ib/bAxZVDo6wy7itpAqVxn5FayN6VwpGyt8YoDH5+q57ePyeQD2YNnPN77fll/z4+Cur0IlgtlMJN/gJQSwMEFAAAAAgAmmGINg5gkmcwAAAASAAAAAoAFQBzaGFwZXMudHh0VVQJAAOzPhlGUT8ZRlV4BAD1ARQAK85ILEiNz0zRKQYzCkricxJLkDj5eQhOcWphaWpecipUJCWzuCS+pCixLDUnNQUAUEsDBBQAAAAIAKJgiDZbqKshjQEAAHsEAAAOABUAc3RvcF90aW1lcy50eHRVVAkAA+A8GUZRPxlGVXgEAPUBFACNk1tvgkAQhd+b+E/2gVnA29uCaE29NIgmPm2IrJXUKgU06b/vArOLQNOWkJyF8zFzdoA8jRMeRyRM0/gennkefwgSiSRM81sqqsssv5ZMqZn4vInLAe+eRBhl8duFJPHh/Zbw/CuRj6fSuR6P+PQpTASP4izneRrexVlEvadN4DDSHxuGPLVuAjbz3DVznwkQeWiMIlap47Eg2HM291/XfkBohbrzYA//KKk5G7lBoSu2YqxdCmiFgFUhE7YjZgsZYTIodCKRFbGaCO0jMizUW27mxK4Jqp3+2DQ0AU3CxLDmoG5Dm4iFYa2fw5YIhrWh3rLVRGwMaw9b08PIzAEyxNkqbb0PeCQBSSg34GwXi6m/nqnszKGkGHJp6oWmoEGBorBW6yvoPTlTt2hJsSV+LLoYAiM0lE63Pnd9z3vhvrfBMrIf4Ob0osmB4qji1KLeo0zOlru/56W4EfpK5U1VRDYyVCPjwQZtd/J251OQJgFTkeaveSRpKdLqJrJI+RpK2+4mkrb6BfWinegbUEsDBBQAAAAIAFpfiDYCyszVWwEAAF0CAAAJABUAc3RvcHMudHh0VVQJAANrOxlGUj8ZRlV4BAD1ARQAfZHLbsIwEEX3lfgHr6pWmlKcOI6zNEl4qBCqBJC6Qha4EDXEKDFI9OvrPNqSLrqau/A5d2yXWp026Q7KaubiKJu0k+W2SZnQbVA5fKpc/pw+F1nvbrSKN34chi+bOExgdC5ysZXIL6T8QLEsVaHRQyCP6hHApn1iORZj8ISx28e2jakF0Lsbhny5fNvwafy6iJcQXY1AnXN9RTwtTn8UjDJCaKWgfZcRhzWK1Ww2ihdjGJ6z7L1Q+w7C8IA1BMOu51ZAsuTj0F9wfwKJFnu5VWJ7QBOlZYbukS/KNFe3Dg871FQ1tQ6mbm2JeMDXEJkVD4hfJHpGQT2jLkqYZ7eokeCGjDjvkBHideqSHiHfJCZufdfAlEYQqDTf/1M68Ezrbymp0XCeTCFEc5HmKNGGTNC0uFSiRHdph5p3amkLswrm8zXwoyj2qhRoLbJMXm8ZSsy27ceQgakH+AJQSwMEFAAAAAgAbmCINvJ/+UPNAAAApQEAAAkAFQB0cmlwcy50eHRVVAkAA4A8GUZSPxlGVXgEAPUBFACNj00OgjAQhfcm3oEDzELQC7REEhPd+ANxZSqM0FgpGYqJt7ctxAhuXM3L+2ZeX0l3Bi+ygBbpKXMvDcnmMysURSvLGgpJmBupa4euSud3f1aJxh3NZ4xDctpuM2A8BKMD3il1I13CAkIY4chhJqnRZCyLLD0cORu4l4eqM0YhgGXx5ngemJMh2MRfOwIbZW2exINrlS+SdFSLHIOYEO/BHlv3bF9qtByNWve9GNulkK3BTR/GHoJK3YogFUrhq6/yvTX53IQu/8pYTTLeUEsBAhcDFAAAAAgAJl+INphs6TZdAAAAcQAAAAoADQAAAAAAAQAAAKSBAAAAAGFnZW5jeS50eHRVVAUAAwc7GUZVeAAAUEsBAhcDFAAAAAgA0GCINjwKTMFlAAAAogAAAAwADQAAAAAAAQAAAKSBmgAAAGNhbGVuZGFyLnR4dFVUBQADNz0ZRlV4AABQSwECFwMKAAAAAAANYYg2I3zXrDAAAAAwAAAAEgANAAAAAAABAAAApIE+AQAAY2FsZW5kYXJfZGF0ZXMudHh0VVQFAAOqPRlGVXgAAFBLAQIXAxQAAAAIAEFhiDZpEYREVgAAAGgAAAATAA0AAAAAAAEAAACkgbMBAABmYXJlX2F0dHJpYnV0ZXMudHh0VVQFAAMJPhlGVXgAAFBLAQIXAxQAAAAIAIZhiDb1AII+SQAAAF4AAAAOAA0AAAAAAAEAAACkgU8CAABmYXJlX3J1bGVzLnR4dFVUBQADjD4ZRlV4AABQSwECFwMUAAAACAC5YYg2OnFgWXcAAABZAQAADwANAAAAAAABAAAApIHZAgAAZnJlcXVlbmNpZXMudHh0VVQFAAPtPhlGVXgAAFBLAQIXAxQAAAAIADJgiDYDlanbvAAAADsBAAAKAA0AAAAAAAEAAACkgZIDAAByb3V0ZXMudHh0VVQFAAMQPBlGVXgAAFBLAQIXAxQAAAAIAJphiDYOYJJnMAAAAEgAAAAKAA0AAAAAAAEAAACkgYsEAABzaGFwZXMudHh0VVQFAAOzPhlGVXgAAFBLAQIXAxQAAAAIAKJgiDZbqKshjQEAAHsEAAAOAA0AAAAAAAEAAACkgfgEAABzdG9wX3RpbWVzLnR4dFVUBQAD4DwZRlV4AABQSwECFwMUAAAACABaX4g2AsrM1VsBAABdAgAACQANAAAAAAABAAAApIHGBgAAc3RvcHMudHh0VVQFAANrOxlGVXgAAFBLAQIXAxQAAAAIAG5giDbyf/lDzQAAAKUBAAAJAA0AAAAAAAEAAACkgV0IAAB0cmlwcy50eHRVVAUAA4A8GUZVeAAAUEsFBgAAAAALAAsAFQMAAGYJAAAAAA== - recorded_at: Fri, 23 Oct 2020 23:02:49 GMT -recorded_with: VCR 6.0.0 diff --git a/spec/gtfs/url_source_spec.rb b/spec/gtfs/url_source_spec.rb index 84ecbed..ddc491f 100644 --- a/spec/gtfs/url_source_spec.rb +++ b/spec/gtfs/url_source_spec.rb @@ -10,6 +10,15 @@ end end + context 'with a URI causing redirects' do + let(:source_path) {'https://api.gtfs-data.jp/v2/organizations/asatetu/feeds/asakaigantetsudo/files/feed.zip'} # 302 Found + it 'should create a new source successfully' do + VCR.use_cassette('redirect_gtfs_uri') do + lambda {GTFS::URLSource.new(source_path, {})}.should_not raise_error(GTFS::InvalidSourceException) + end + end + end + context 'with a non-existent URI' do let(:source_path) {'https://example.org/gtfs.zip'} it 'should raise an exception' do