From 663eebc0807575a4b52635d820d6720909d4f74a Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Fri, 16 Sep 2016 11:10:54 +0200 Subject: [PATCH] Add multipart file upload to PostAgent This allows the PostAgent to receive `file_pointer` events and upload the data to a website. --- app/concerns/file_handling.rb | 14 +++- app/concerns/web_request_concern.rb | 1 + app/models/agents/post_agent.rb | 73 ++++++++++++------- spec/models/agents/post_agent_spec.rb | 18 +++++ .../shared_examples/file_handling_consumer.rb | 26 ++++++- 5 files changed, 98 insertions(+), 34 deletions(-) diff --git a/app/concerns/file_handling.rb b/app/concerns/file_handling.rb index 261e1285..c4f43882 100644 --- a/app/concerns/file_handling.rb +++ b/app/concerns/file_handling.rb @@ -5,13 +5,21 @@ module FileHandling { file_pointer: { file: file, agent_id: id } } end + def has_file_pointer?(event) + event.payload['file_pointer'] && + event.payload['file_pointer']['file'] && + event.payload['file_pointer']['agent_id'] + end + def get_io(event) - return nil unless event.payload['file_pointer'] && - event.payload['file_pointer']['file'] && - event.payload['file_pointer']['agent_id'] + return nil unless has_file_pointer?(event) event.user.agents.find(event.payload['file_pointer']['agent_id']).get_io(event.payload['file_pointer']['file']) end + def get_upload_io(event) + Faraday::UploadIO.new(get_io(event), MIME::Types.type_for(File.basename(event.payload['file_pointer']['file'])).first.try(:content_type)) + end + def emitting_file_handling_agent_description @emitting_file_handling_agent_description ||= "This agent only emits a 'file pointer', not the data inside the files, the following agents can consume the created events: `#{receiving_file_handling_agents.join('`, `')}`. Read more about the concept in the [wiki](https://github.com/cantino/huginn/wiki/How-Huginn-works-with-files)." diff --git a/app/concerns/web_request_concern.rb b/app/concerns/web_request_concern.rb index ef6adc4a..71a76427 100644 --- a/app/concerns/web_request_concern.rb +++ b/app/concerns/web_request_concern.rb @@ -113,6 +113,7 @@ module WebRequestConcern unless boolify(interpolated['disable_redirect_follow']) builder.use FaradayMiddleware::FollowRedirects end + builder.request :multipart builder.request :url_encoded if boolify(interpolated['disable_url_encoding']) diff --git a/app/models/agents/post_agent.rb b/app/models/agents/post_agent.rb index a6e754d0..d31c0233 100644 --- a/app/models/agents/post_agent.rb +++ b/app/models/agents/post_agent.rb @@ -1,6 +1,9 @@ module Agents class PostAgent < Agent include WebRequestConcern + include FileHandling + + consumes_file_pointer! MIME_RE = /\A\w+\/.+\z/ @@ -8,38 +11,44 @@ module Agents no_bulk_receive! default_schedule "never" - description <<-MD - A Post Agent receives events from other agents (or runs periodically), merges those events with the [Liquid-interpolated](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) contents of `payload`, and sends the results as POST (or GET) requests to a specified url. To skip merging in the incoming event, but still send the interpolated payload, set `no_merge` to `true`. + description do + <<-MD + A Post Agent receives events from other agents (or runs periodically), merges those events with the [Liquid-interpolated](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) contents of `payload`, and sends the results as POST (or GET) requests to a specified url. To skip merging in the incoming event, but still send the interpolated payload, set `no_merge` to `true`. - The `post_url` field must specify where you would like to send requests. Please include the URI scheme (`http` or `https`). + The `post_url` field must specify where you would like to send requests. Please include the URI scheme (`http` or `https`). - The `method` used can be any of `get`, `post`, `put`, `patch`, and `delete`. + The `method` used can be any of `get`, `post`, `put`, `patch`, and `delete`. - By default, non-GETs will be sent with form encoding (`application/x-www-form-urlencoded`). + By default, non-GETs will be sent with form encoding (`application/x-www-form-urlencoded`). - Change `content_type` to `json` to send JSON instead. + Change `content_type` to `json` to send JSON instead. - Change `content_type` to `xml` to send XML, where the name of the root element may be specified using `xml_root`, defaulting to `post`. + Change `content_type` to `xml` to send XML, where the name of the root element may be specified using `xml_root`, defaulting to `post`. - When `content_type` contains a [MIME](https://en.wikipedia.org/wiki/Media_type) type, and `payload` is a string, its interpolated value will be sent as a string in the HTTP request's body and the request's `Content-Type` HTTP header will be set to `content_type`. When `payload` is a string `no_merge` has to be set to `true`. + When `content_type` contains a [MIME](https://en.wikipedia.org/wiki/Media_type) type, and `payload` is a string, its interpolated value will be sent as a string in the HTTP request's body and the request's `Content-Type` HTTP header will be set to `content_type`. When `payload` is a string `no_merge` has to be set to `true`. - If `emit_events` is set to `true`, the server response will be emitted as an Event and can be fed to a WebsiteAgent for parsing (using its `data_from_event` and `type` options). No data processing - will be attempted by this Agent, so the Event's "body" value will always be raw text. - The Event will also have a "headers" hash and a "status" integer value. - Set `event_headers_style` to one of the following values to normalize the keys of "headers" for downstream agents' convenience: + If `emit_events` is set to `true`, the server response will be emitted as an Event and can be fed to a WebsiteAgent for parsing (using its `data_from_event` and `type` options). No data processing + will be attempted by this Agent, so the Event's "body" value will always be raw text. + The Event will also have a "headers" hash and a "status" integer value. + Set `event_headers_style` to one of the following values to normalize the keys of "headers" for downstream agents' convenience: - * `capitalized` (default) - Header names are capitalized; e.g. "Content-Type" - * `downcased` - Header names are downcased; e.g. "content-type" - * `snakecased` - Header names are snakecased; e.g. "content_type" - * `raw` - Backward compatibility option to leave them unmodified from what the underlying HTTP library returns. + * `capitalized` (default) - Header names are capitalized; e.g. "Content-Type" + * `downcased` - Header names are downcased; e.g. "content-type" + * `snakecased` - Header names are snakecased; e.g. "content_type" + * `raw` - Backward compatibility option to leave them unmodified from what the underlying HTTP library returns. - Other Options: + Other Options: - * `headers` - When present, it should be a hash of headers to send with the request. - * `basic_auth` - Specify HTTP basic auth parameters: `"username:password"`, or `["username", "password"]`. - * `disable_ssl_verification` - Set to `true` to disable ssl verification. - * `user_agent` - A custom User-Agent name (default: "Faraday v#{Faraday::VERSION}"). - MD + * `headers` - When present, it should be a hash of headers to send with the request. + * `basic_auth` - Specify HTTP basic auth parameters: `"username:password"`, or `["username", "password"]`. + * `disable_ssl_verification` - Set to `true` to disable ssl verification. + * `user_agent` - A custom User-Agent name (default: "Faraday v#{Faraday::VERSION}"). + + #{receiving_file_handling_agent_description} + + When receiving a `file_pointer` the request will be sent with multipart encoding (`multipart/form-data`) and `content_type` is ignored. `upload_key` can be used to specify the parameter in which the file will be sent, it defaults to `file`. + MD + end event_description <<-MD Events look like this: @@ -125,9 +134,9 @@ module Agents interpolate_with(event) do outgoing = interpolated['payload'].presence || {} if boolify(interpolated['no_merge']) - handle outgoing, event.payload, headers(interpolated[:headers]) + handle outgoing, event, headers(interpolated[:headers]) else - handle outgoing.merge(event.payload), event.payload, headers(interpolated[:headers]) + handle outgoing.merge(event.payload), event, headers(interpolated[:headers]) end end end @@ -162,8 +171,8 @@ module Agents } end - def handle(data, payload = {}, headers) - url = interpolated(payload)[:post_url] + def handle(data, event = Event.new, headers) + url = interpolated(event.payload)[:post_url] case method when 'get', 'delete' @@ -171,13 +180,21 @@ module Agents when 'post', 'put', 'patch' params = nil - case (content_type = interpolated(payload)['content_type']) + content_type = + if has_file_pointer?(event) + data[interpolated(event.payload)['upload_key'].presence || 'file'] = get_upload_io(event) + nil + else + interpolated(event.payload)['content_type'] + end + + case content_type when 'json' headers['Content-Type'] = 'application/json; charset=utf-8' body = data.to_json when 'xml' headers['Content-Type'] = 'text/xml; charset=utf-8' - body = data.to_xml(root: (interpolated(payload)[:xml_root] || 'post')) + body = data.to_xml(root: (interpolated(event.payload)[:xml_root] || 'post')) when MIME_RE headers['Content-Type'] = content_type body = data.to_s diff --git a/spec/models/agents/post_agent_spec.rb b/spec/models/agents/post_agent_spec.rb index e940d330..e811f8a1 100644 --- a/spec/models/agents/post_agent_spec.rb +++ b/spec/models/agents/post_agent_spec.rb @@ -57,6 +57,11 @@ describe Agents::PostAgent do end it_behaves_like WebRequestConcern + it_behaves_like 'FileHandlingConsumer' + + it 'renders the description markdown without errors' do + expect { @checker.description }.not_to raise_error + end describe "making requests" do it "can make requests of each type" do @@ -149,6 +154,19 @@ describe Agents::PostAgent do headers = @sent_requests[:post].first.headers expect(headers["Foo"]).to eq("a_variable") end + + it 'makes a multipart request when receiving a file_pointer' do + WebMock.reset! + stub_request(:post, "http://www.example.com/"). + with(:body => "-------------RubyMultipartPost\r\nContent-Disposition: form-data; name=\"default\"\r\n\r\nvalue\r\n-------------RubyMultipartPost\r\nContent-Disposition: form-data; name=\"file\"; filename=\"local.path\"\r\nContent-Length: 8\r\nContent-Type: \r\nContent-Transfer-Encoding: binary\r\n\r\ntestdata\r\n-------------RubyMultipartPost--\r\n\r\n", + :headers => {'Accept-Encoding'=>'gzip,deflate', 'Content-Length'=>'307', 'Content-Type'=>'multipart/form-data; boundary=-----------RubyMultipartPost', 'User-Agent'=>'Huginn - https://github.com/cantino/huginn'}). + to_return(:status => 200, :body => "", :headers => {}) + event = Event.new(payload: {file_pointer: {agent_id: 111, file: 'test'}}) + io_mock = mock() + mock(@checker).get_io(event) { StringIO.new("testdata") } + @checker.options['no_merge'] = true + @checker.receive([event]) + end end describe "#check" do diff --git a/spec/support/shared_examples/file_handling_consumer.rb b/spec/support/shared_examples/file_handling_consumer.rb index a864bb2e..67bb1f1c 100644 --- a/spec/support/shared_examples/file_handling_consumer.rb +++ b/spec/support/shared_examples/file_handling_consumer.rb @@ -1,6 +1,8 @@ require 'rails_helper' shared_examples_for 'FileHandlingConsumer' do + let(:event) { Event.new(user: @checker.user, payload: {'file_pointer' => {'file' => 'text.txt', 'agent_id' => @checker.id}}) } + it 'returns a file pointer' do expect(@checker.get_file_pointer('testfile')).to eq(file_pointer: { file: "testfile", agent_id: @checker.id}) end @@ -9,8 +11,26 @@ shared_examples_for 'FileHandlingConsumer' do @checker2 = @checker.dup @checker2.user = users(:bob) @checker2.save! - expect(@checker2.user.id).not_to eq(@checker.user.id) - event = Event.new(user: @checker.user, payload: {'file_pointer' => {'file' => 'test', 'agent_id' => @checker2.id}}) + event.payload['file_pointer']['agent_id'] = @checker2.id expect { @checker.get_io(event) }.to raise_error(ActiveRecord::RecordNotFound) end -end \ No newline at end of file + + context '#has_file_pointer?' do + it 'returns true if the event contains a file pointer' do + expect(@checker.has_file_pointer?(event)).to be_truthy + end + + it 'returns false if the event does not contain a file pointer' do + expect(@checker.has_file_pointer?(Event.new)).to be_falsy + end + end + + it '#get_upload_io returns a Faraday::UploadIO instance' do + io_mock = mock() + mock(@checker).get_io(event) { StringIO.new("testdata") } + + upload_io = @checker.get_upload_io(event) + expect(upload_io).to be_a(Faraday::UploadIO) + expect(upload_io.content_type).to eq('text/plain') + end +end