mirror of
https://github.com/Fishwaldo/huginn.git
synced 2025-03-15 19:31:26 +00:00
Add multipart file upload to PostAgent
This allows the PostAgent to receive `file_pointer` events and upload the data to a website.
This commit is contained in:
parent
4fdd69c6f4
commit
663eebc080
5 changed files with 98 additions and 34 deletions
|
@ -5,13 +5,21 @@ module FileHandling
|
|||
{ file_pointer: { file: file, agent_id: id } }
|
||||
end
|
||||
|
||||
def has_file_pointer?(event)
|
||||
event.payload['file_pointer'] &&
|
||||
event.payload['file_pointer']['file'] &&
|
||||
event.payload['file_pointer']['agent_id']
|
||||
end
|
||||
|
||||
def get_io(event)
|
||||
return nil unless event.payload['file_pointer'] &&
|
||||
event.payload['file_pointer']['file'] &&
|
||||
event.payload['file_pointer']['agent_id']
|
||||
return nil unless has_file_pointer?(event)
|
||||
event.user.agents.find(event.payload['file_pointer']['agent_id']).get_io(event.payload['file_pointer']['file'])
|
||||
end
|
||||
|
||||
def get_upload_io(event)
|
||||
Faraday::UploadIO.new(get_io(event), MIME::Types.type_for(File.basename(event.payload['file_pointer']['file'])).first.try(:content_type))
|
||||
end
|
||||
|
||||
def emitting_file_handling_agent_description
|
||||
@emitting_file_handling_agent_description ||=
|
||||
"This agent only emits a 'file pointer', not the data inside the files, the following agents can consume the created events: `#{receiving_file_handling_agents.join('`, `')}`. Read more about the concept in the [wiki](https://github.com/cantino/huginn/wiki/How-Huginn-works-with-files)."
|
||||
|
|
|
@ -113,6 +113,7 @@ module WebRequestConcern
|
|||
unless boolify(interpolated['disable_redirect_follow'])
|
||||
builder.use FaradayMiddleware::FollowRedirects
|
||||
end
|
||||
builder.request :multipart
|
||||
builder.request :url_encoded
|
||||
|
||||
if boolify(interpolated['disable_url_encoding'])
|
||||
|
|
|
@ -1,6 +1,9 @@
|
|||
module Agents
|
||||
class PostAgent < Agent
|
||||
include WebRequestConcern
|
||||
include FileHandling
|
||||
|
||||
consumes_file_pointer!
|
||||
|
||||
MIME_RE = /\A\w+\/.+\z/
|
||||
|
||||
|
@ -8,38 +11,44 @@ module Agents
|
|||
no_bulk_receive!
|
||||
default_schedule "never"
|
||||
|
||||
description <<-MD
|
||||
A Post Agent receives events from other agents (or runs periodically), merges those events with the [Liquid-interpolated](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) contents of `payload`, and sends the results as POST (or GET) requests to a specified url. To skip merging in the incoming event, but still send the interpolated payload, set `no_merge` to `true`.
|
||||
description do
|
||||
<<-MD
|
||||
A Post Agent receives events from other agents (or runs periodically), merges those events with the [Liquid-interpolated](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) contents of `payload`, and sends the results as POST (or GET) requests to a specified url. To skip merging in the incoming event, but still send the interpolated payload, set `no_merge` to `true`.
|
||||
|
||||
The `post_url` field must specify where you would like to send requests. Please include the URI scheme (`http` or `https`).
|
||||
The `post_url` field must specify where you would like to send requests. Please include the URI scheme (`http` or `https`).
|
||||
|
||||
The `method` used can be any of `get`, `post`, `put`, `patch`, and `delete`.
|
||||
The `method` used can be any of `get`, `post`, `put`, `patch`, and `delete`.
|
||||
|
||||
By default, non-GETs will be sent with form encoding (`application/x-www-form-urlencoded`).
|
||||
By default, non-GETs will be sent with form encoding (`application/x-www-form-urlencoded`).
|
||||
|
||||
Change `content_type` to `json` to send JSON instead.
|
||||
Change `content_type` to `json` to send JSON instead.
|
||||
|
||||
Change `content_type` to `xml` to send XML, where the name of the root element may be specified using `xml_root`, defaulting to `post`.
|
||||
Change `content_type` to `xml` to send XML, where the name of the root element may be specified using `xml_root`, defaulting to `post`.
|
||||
|
||||
When `content_type` contains a [MIME](https://en.wikipedia.org/wiki/Media_type) type, and `payload` is a string, its interpolated value will be sent as a string in the HTTP request's body and the request's `Content-Type` HTTP header will be set to `content_type`. When `payload` is a string `no_merge` has to be set to `true`.
|
||||
When `content_type` contains a [MIME](https://en.wikipedia.org/wiki/Media_type) type, and `payload` is a string, its interpolated value will be sent as a string in the HTTP request's body and the request's `Content-Type` HTTP header will be set to `content_type`. When `payload` is a string `no_merge` has to be set to `true`.
|
||||
|
||||
If `emit_events` is set to `true`, the server response will be emitted as an Event and can be fed to a WebsiteAgent for parsing (using its `data_from_event` and `type` options). No data processing
|
||||
will be attempted by this Agent, so the Event's "body" value will always be raw text.
|
||||
The Event will also have a "headers" hash and a "status" integer value.
|
||||
Set `event_headers_style` to one of the following values to normalize the keys of "headers" for downstream agents' convenience:
|
||||
If `emit_events` is set to `true`, the server response will be emitted as an Event and can be fed to a WebsiteAgent for parsing (using its `data_from_event` and `type` options). No data processing
|
||||
will be attempted by this Agent, so the Event's "body" value will always be raw text.
|
||||
The Event will also have a "headers" hash and a "status" integer value.
|
||||
Set `event_headers_style` to one of the following values to normalize the keys of "headers" for downstream agents' convenience:
|
||||
|
||||
* `capitalized` (default) - Header names are capitalized; e.g. "Content-Type"
|
||||
* `downcased` - Header names are downcased; e.g. "content-type"
|
||||
* `snakecased` - Header names are snakecased; e.g. "content_type"
|
||||
* `raw` - Backward compatibility option to leave them unmodified from what the underlying HTTP library returns.
|
||||
* `capitalized` (default) - Header names are capitalized; e.g. "Content-Type"
|
||||
* `downcased` - Header names are downcased; e.g. "content-type"
|
||||
* `snakecased` - Header names are snakecased; e.g. "content_type"
|
||||
* `raw` - Backward compatibility option to leave them unmodified from what the underlying HTTP library returns.
|
||||
|
||||
Other Options:
|
||||
Other Options:
|
||||
|
||||
* `headers` - When present, it should be a hash of headers to send with the request.
|
||||
* `basic_auth` - Specify HTTP basic auth parameters: `"username:password"`, or `["username", "password"]`.
|
||||
* `disable_ssl_verification` - Set to `true` to disable ssl verification.
|
||||
* `user_agent` - A custom User-Agent name (default: "Faraday v#{Faraday::VERSION}").
|
||||
MD
|
||||
* `headers` - When present, it should be a hash of headers to send with the request.
|
||||
* `basic_auth` - Specify HTTP basic auth parameters: `"username:password"`, or `["username", "password"]`.
|
||||
* `disable_ssl_verification` - Set to `true` to disable ssl verification.
|
||||
* `user_agent` - A custom User-Agent name (default: "Faraday v#{Faraday::VERSION}").
|
||||
|
||||
#{receiving_file_handling_agent_description}
|
||||
|
||||
When receiving a `file_pointer` the request will be sent with multipart encoding (`multipart/form-data`) and `content_type` is ignored. `upload_key` can be used to specify the parameter in which the file will be sent, it defaults to `file`.
|
||||
MD
|
||||
end
|
||||
|
||||
event_description <<-MD
|
||||
Events look like this:
|
||||
|
@ -125,9 +134,9 @@ module Agents
|
|||
interpolate_with(event) do
|
||||
outgoing = interpolated['payload'].presence || {}
|
||||
if boolify(interpolated['no_merge'])
|
||||
handle outgoing, event.payload, headers(interpolated[:headers])
|
||||
handle outgoing, event, headers(interpolated[:headers])
|
||||
else
|
||||
handle outgoing.merge(event.payload), event.payload, headers(interpolated[:headers])
|
||||
handle outgoing.merge(event.payload), event, headers(interpolated[:headers])
|
||||
end
|
||||
end
|
||||
end
|
||||
|
@ -162,8 +171,8 @@ module Agents
|
|||
}
|
||||
end
|
||||
|
||||
def handle(data, payload = {}, headers)
|
||||
url = interpolated(payload)[:post_url]
|
||||
def handle(data, event = Event.new, headers)
|
||||
url = interpolated(event.payload)[:post_url]
|
||||
|
||||
case method
|
||||
when 'get', 'delete'
|
||||
|
@ -171,13 +180,21 @@ module Agents
|
|||
when 'post', 'put', 'patch'
|
||||
params = nil
|
||||
|
||||
case (content_type = interpolated(payload)['content_type'])
|
||||
content_type =
|
||||
if has_file_pointer?(event)
|
||||
data[interpolated(event.payload)['upload_key'].presence || 'file'] = get_upload_io(event)
|
||||
nil
|
||||
else
|
||||
interpolated(event.payload)['content_type']
|
||||
end
|
||||
|
||||
case content_type
|
||||
when 'json'
|
||||
headers['Content-Type'] = 'application/json; charset=utf-8'
|
||||
body = data.to_json
|
||||
when 'xml'
|
||||
headers['Content-Type'] = 'text/xml; charset=utf-8'
|
||||
body = data.to_xml(root: (interpolated(payload)[:xml_root] || 'post'))
|
||||
body = data.to_xml(root: (interpolated(event.payload)[:xml_root] || 'post'))
|
||||
when MIME_RE
|
||||
headers['Content-Type'] = content_type
|
||||
body = data.to_s
|
||||
|
|
|
@ -57,6 +57,11 @@ describe Agents::PostAgent do
|
|||
end
|
||||
|
||||
it_behaves_like WebRequestConcern
|
||||
it_behaves_like 'FileHandlingConsumer'
|
||||
|
||||
it 'renders the description markdown without errors' do
|
||||
expect { @checker.description }.not_to raise_error
|
||||
end
|
||||
|
||||
describe "making requests" do
|
||||
it "can make requests of each type" do
|
||||
|
@ -149,6 +154,19 @@ describe Agents::PostAgent do
|
|||
headers = @sent_requests[:post].first.headers
|
||||
expect(headers["Foo"]).to eq("a_variable")
|
||||
end
|
||||
|
||||
it 'makes a multipart request when receiving a file_pointer' do
|
||||
WebMock.reset!
|
||||
stub_request(:post, "http://www.example.com/").
|
||||
with(:body => "-------------RubyMultipartPost\r\nContent-Disposition: form-data; name=\"default\"\r\n\r\nvalue\r\n-------------RubyMultipartPost\r\nContent-Disposition: form-data; name=\"file\"; filename=\"local.path\"\r\nContent-Length: 8\r\nContent-Type: \r\nContent-Transfer-Encoding: binary\r\n\r\ntestdata\r\n-------------RubyMultipartPost--\r\n\r\n",
|
||||
:headers => {'Accept-Encoding'=>'gzip,deflate', 'Content-Length'=>'307', 'Content-Type'=>'multipart/form-data; boundary=-----------RubyMultipartPost', 'User-Agent'=>'Huginn - https://github.com/cantino/huginn'}).
|
||||
to_return(:status => 200, :body => "", :headers => {})
|
||||
event = Event.new(payload: {file_pointer: {agent_id: 111, file: 'test'}})
|
||||
io_mock = mock()
|
||||
mock(@checker).get_io(event) { StringIO.new("testdata") }
|
||||
@checker.options['no_merge'] = true
|
||||
@checker.receive([event])
|
||||
end
|
||||
end
|
||||
|
||||
describe "#check" do
|
||||
|
|
|
@ -1,6 +1,8 @@
|
|||
require 'rails_helper'
|
||||
|
||||
shared_examples_for 'FileHandlingConsumer' do
|
||||
let(:event) { Event.new(user: @checker.user, payload: {'file_pointer' => {'file' => 'text.txt', 'agent_id' => @checker.id}}) }
|
||||
|
||||
it 'returns a file pointer' do
|
||||
expect(@checker.get_file_pointer('testfile')).to eq(file_pointer: { file: "testfile", agent_id: @checker.id})
|
||||
end
|
||||
|
@ -9,8 +11,26 @@ shared_examples_for 'FileHandlingConsumer' do
|
|||
@checker2 = @checker.dup
|
||||
@checker2.user = users(:bob)
|
||||
@checker2.save!
|
||||
expect(@checker2.user.id).not_to eq(@checker.user.id)
|
||||
event = Event.new(user: @checker.user, payload: {'file_pointer' => {'file' => 'test', 'agent_id' => @checker2.id}})
|
||||
event.payload['file_pointer']['agent_id'] = @checker2.id
|
||||
expect { @checker.get_io(event) }.to raise_error(ActiveRecord::RecordNotFound)
|
||||
end
|
||||
end
|
||||
|
||||
context '#has_file_pointer?' do
|
||||
it 'returns true if the event contains a file pointer' do
|
||||
expect(@checker.has_file_pointer?(event)).to be_truthy
|
||||
end
|
||||
|
||||
it 'returns false if the event does not contain a file pointer' do
|
||||
expect(@checker.has_file_pointer?(Event.new)).to be_falsy
|
||||
end
|
||||
end
|
||||
|
||||
it '#get_upload_io returns a Faraday::UploadIO instance' do
|
||||
io_mock = mock()
|
||||
mock(@checker).get_io(event) { StringIO.new("testdata") }
|
||||
|
||||
upload_io = @checker.get_upload_io(event)
|
||||
expect(upload_io).to be_a(Faraday::UploadIO)
|
||||
expect(upload_io.content_type).to eq('text/plain')
|
||||
end
|
||||
end
|
||||
|
|
Loading…
Add table
Reference in a new issue