GitHub-Mirror/huginn
2
0
Fork 0
mirror of https://github.com/Fishwaldo/huginn.git synced 2025-03-17 12:21:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add multipart file upload to PostAgent

Browse source 
This allows the PostAgent to receive `file_pointer` events and upload the data to a website.
This commit is contained in:
Dominik Sander 2016-09-16 11:10:54 +02:00
parent 4fdd69c6f4
commit 663eebc080
5 changed files with 98 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
app/concerns/file_handling.rb
View file

@ -5,13 +5,21 @@ module FileHandling
{ file_pointer: { file: file, agent_id: id } }
end
def has_file_pointer?(event)
event.payload['file_pointer'] &&
event.payload['file_pointer']['file'] &&
event.payload['file_pointer']['agent_id']
end
def get_io(event)
return nil unless event.payload['file_pointer'] &&
event.payload['file_pointer']['file'] &&
event.payload['file_pointer']['agent_id']
return nil unless has_file_pointer?(event)
event.user.agents.find(event.payload['file_pointer']['agent_id']).get_io(event.payload['file_pointer']['file'])
end
def get_upload_io(event)
Faraday::UploadIO.new(get_io(event), MIME::Types.type_for(File.basename(event.payload['file_pointer']['file'])).first.try(:content_type))
end
def emitting_file_handling_agent_description
@emitting_file_handling_agent_description ||=
"This agent only emits a 'file pointer', not the data inside the files, the following agents can consume the created events: `#{receiving_file_handling_agents.join('`, `')}`. Read more about the concept in the [wiki](https://github.com/cantino/huginn/wiki/How-Huginn-works-with-files)."

1
app/concerns/web_request_concern.rb
View file

@ -113,6 +113,7 @@ module WebRequestConcern
unless boolify(interpolated['disable_redirect_follow'])
builder.use FaradayMiddleware::FollowRedirects
end
builder.request :multipart
builder.request :url_encoded
if boolify(interpolated['disable_url_encoding'])

73
app/models/agents/post_agent.rb
View file

@ -1,6 +1,9 @@
module Agents
class PostAgent < Agent
include WebRequestConcern
include FileHandling
consumes_file_pointer!
MIME_RE = /\A\w+\/.+\z/
@ -8,38 +11,44 @@ module Agents
no_bulk_receive!
default_schedule "never"
description <<-MD
A Post Agent receives events from other agents (or runs periodically), merges those events with the [Liquid-interpolated](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) contents of `payload`, and sends the results as POST (or GET) requests to a specified url. To skip merging in the incoming event, but still send the interpolated payload, set `no_merge` to `true`.
description do
<<-MD
A Post Agent receives events from other agents (or runs periodically), merges those events with the [Liquid-interpolated](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) contents of `payload`, and sends the results as POST (or GET) requests to a specified url. To skip merging in the incoming event, but still send the interpolated payload, set `no_merge` to `true`.
The `post_url` field must specify where you would like to send requests. Please include the URI scheme (`http` or `https`).
The `post_url` field must specify where you would like to send requests. Please include the URI scheme (`http` or `https`).
The `method` used can be any of `get`, `post`, `put`, `patch`, and `delete`.
The `method` used can be any of `get`, `post`, `put`, `patch`, and `delete`.
By default, non-GETs will be sent with form encoding (`application/x-www-form-urlencoded`).
By default, non-GETs will be sent with form encoding (`application/x-www-form-urlencoded`).
Change `content_type` to `json` to send JSON instead.
Change `content_type` to `json` to send JSON instead.
Change `content_type` to `xml` to send XML, where the name of the root element may be specified using `xml_root`, defaulting to `post`.
Change `content_type` to `xml` to send XML, where the name of the root element may be specified using `xml_root`, defaulting to `post`.
When `content_type` contains a [MIME](https://en.wikipedia.org/wiki/Media_type) type, and `payload` is a string, its interpolated value will be sent as a string in the HTTP request's body and the request's `Content-Type` HTTP header will be set to `content_type`. When `payload` is a string `no_merge` has to be set to `true`.
When `content_type` contains a [MIME](https://en.wikipedia.org/wiki/Media_type) type, and `payload` is a string, its interpolated value will be sent as a string in the HTTP request's body and the request's `Content-Type` HTTP header will be set to `content_type`. When `payload` is a string `no_merge` has to be set to `true`.
If `emit_events` is set to `true`, the server response will be emitted as an Event and can be fed to a WebsiteAgent for parsing (using its `data_from_event` and `type` options). No data processing
will be attempted by this Agent, so the Event's "body" value will always be raw text.
The Event will also have a "headers" hash and a "status" integer value.
Set `event_headers_style` to one of the following values to normalize the keys of "headers" for downstream agents' convenience:
If `emit_events` is set to `true`, the server response will be emitted as an Event and can be fed to a WebsiteAgent for parsing (using its `data_from_event` and `type` options). No data processing
will be attempted by this Agent, so the Event's "body" value will always be raw text.
The Event will also have a "headers" hash and a "status" integer value.
Set `event_headers_style` to one of the following values to normalize the keys of "headers" for downstream agents' convenience:
* `capitalized` (default) - Header names are capitalized; e.g. "Content-Type"
* `downcased` - Header names are downcased; e.g. "content-type"
* `snakecased` - Header names are snakecased; e.g. "content_type"
* `raw` - Backward compatibility option to leave them unmodified from what the underlying HTTP library returns.
* `capitalized` (default) - Header names are capitalized; e.g. "Content-Type"
* `downcased` - Header names are downcased; e.g. "content-type"
* `snakecased` - Header names are snakecased; e.g. "content_type"
* `raw` - Backward compatibility option to leave them unmodified from what the underlying HTTP library returns.
Other Options:
Other Options:
* `headers` - When present, it should be a hash of headers to send with the request.
* `basic_auth` - Specify HTTP basic auth parameters: `"username:password"`, or `["username", "password"]`.
* `disable_ssl_verification` - Set to `true` to disable ssl verification.
* `user_agent` - A custom User-Agent name (default: "Faraday v#{Faraday::VERSION}").
MD
* `headers` - When present, it should be a hash of headers to send with the request.
* `basic_auth` - Specify HTTP basic auth parameters: `"username:password"`, or `["username", "password"]`.
* `disable_ssl_verification` - Set to `true` to disable ssl verification.
* `user_agent` - A custom User-Agent name (default: "Faraday v#{Faraday::VERSION}").
#{receiving_file_handling_agent_description}
When receiving a `file_pointer` the request will be sent with multipart encoding (`multipart/form-data`) and `content_type` is ignored. `upload_key` can be used to specify the parameter in which the file will be sent, it defaults to `file`.
MD
end
event_description <<-MD
Events look like this:
@ -125,9 +134,9 @@ module Agents
interpolate_with(event) do
outgoing = interpolated['payload'].presence || {}
if boolify(interpolated['no_merge'])
handle outgoing, event.payload, headers(interpolated[:headers])
handle outgoing, event, headers(interpolated[:headers])
else
handle outgoing.merge(event.payload), event.payload, headers(interpolated[:headers])
handle outgoing.merge(event.payload), event, headers(interpolated[:headers])
end
end
end
@ -162,8 +171,8 @@ module Agents
}
end
def handle(data, payload = {}, headers)
url = interpolated(payload)[:post_url]
def handle(data, event = Event.new, headers)
url = interpolated(event.payload)[:post_url]
case method
when 'get', 'delete'
@ -171,13 +180,21 @@ module Agents
when 'post', 'put', 'patch'
params = nil
case (content_type = interpolated(payload)['content_type'])
content_type =
if has_file_pointer?(event)
data[interpolated(event.payload)['upload_key'].presence || 'file'] = get_upload_io(event)
nil
else
interpolated(event.payload)['content_type']
end
case content_type
when 'json'
headers['Content-Type'] = 'application/json; charset=utf-8'
body = data.to_json
when 'xml'
headers['Content-Type'] = 'text/xml; charset=utf-8'
body = data.to_xml(root: (interpolated(payload)[:xml_root] || 'post'))
body = data.to_xml(root: (interpolated(event.payload)[:xml_root] || 'post'))
when MIME_RE
headers['Content-Type'] = content_type
body = data.to_s

18
spec/models/agents/post_agent_spec.rb
View file

@ -57,6 +57,11 @@ describe Agents::PostAgent do
end
it_behaves_like WebRequestConcern
it_behaves_like 'FileHandlingConsumer'
it 'renders the description markdown without errors' do
expect { @checker.description }.not_to raise_error
end
describe "making requests" do
it "can make requests of each type" do
@ -149,6 +154,19 @@ describe Agents::PostAgent do
headers = @sent_requests[:post].first.headers
expect(headers["Foo"]).to eq("a_variable")
end
it 'makes a multipart request when receiving a file_pointer' do
WebMock.reset!
stub_request(:post, "http://www.example.com/").
with(:body => "-------------RubyMultipartPost\r\nContent-Disposition: form-data; name=\"default\"\r\n\r\nvalue\r\n-------------RubyMultipartPost\r\nContent-Disposition: form-data; name=\"file\"; filename=\"local.path\"\r\nContent-Length: 8\r\nContent-Type: \r\nContent-Transfer-Encoding: binary\r\n\r\ntestdata\r\n-------------RubyMultipartPost--\r\n\r\n",
:headers => {'Accept-Encoding'=>'gzip,deflate', 'Content-Length'=>'307', 'Content-Type'=>'multipart/form-data; boundary=-----------RubyMultipartPost', 'User-Agent'=>'Huginn - https://github.com/cantino/huginn'}).
to_return(:status => 200, :body => "", :headers => {})
event = Event.new(payload: {file_pointer: {agent_id: 111, file: 'test'}})
io_mock = mock()
mock(@checker).get_io(event) { StringIO.new("testdata") }
@checker.options['no_merge'] = true
@checker.receive([event])
end
end
describe "#check" do

26
spec/support/shared_examples/file_handling_consumer.rb
View file

@ -1,6 +1,8 @@
require 'rails_helper'
shared_examples_for 'FileHandlingConsumer' do
let(:event) { Event.new(user: @checker.user, payload: {'file_pointer' => {'file' => 'text.txt', 'agent_id' => @checker.id}}) }
it 'returns a file pointer' do
expect(@checker.get_file_pointer('testfile')).to eq(file_pointer: { file: "testfile", agent_id: @checker.id})
end
@ -9,8 +11,26 @@ shared_examples_for 'FileHandlingConsumer' do
@checker2 = @checker.dup
@checker2.user = users(:bob)
@checker2.save!
expect(@checker2.user.id).not_to eq(@checker.user.id)
event = Event.new(user: @checker.user, payload: {'file_pointer' => {'file' => 'test', 'agent_id' => @checker2.id}})
event.payload['file_pointer']['agent_id'] = @checker2.id
expect { @checker.get_io(event) }.to raise_error(ActiveRecord::RecordNotFound)
end
end
context '#has_file_pointer?' do
it 'returns true if the event contains a file pointer' do
expect(@checker.has_file_pointer?(event)).to be_truthy
end
it 'returns false if the event does not contain a file pointer' do
expect(@checker.has_file_pointer?(Event.new)).to be_falsy
end
end
it '#get_upload_io returns a Faraday::UploadIO instance' do
io_mock = mock()
mock(@checker).get_io(event) { StringIO.new("testdata") }
upload_io = @checker.get_upload_io(event)
expect(upload_io).to be_a(Faraday::UploadIO)
expect(upload_io.content_type).to eq('text/plain')
end
end