diff --git a/metafacture-io/src/main/java/org/metafacture/io/AbstractUrlConnection.java b/metafacture-io/src/main/java/org/metafacture/io/AbstractUrlConnection.java new file mode 100644 index 000000000..497880ae4 --- /dev/null +++ b/metafacture-io/src/main/java/org/metafacture/io/AbstractUrlConnection.java @@ -0,0 +1,137 @@ +package org.metafacture.io; + +import org.metafacture.framework.ObjectReceiver; +import org.metafacture.framework.helpers.DefaultObjectPipe; + +import java.io.IOException; +import java.net.URL; +import java.net.URLConnection; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Pattern; + +/** + * Common functions for object writers. + * + * @param object type that this module processes + * @param receiver type of the downstream module + * @author Jens Wille + * @author Pascal Christoph (dr0i) + */ +abstract class AbstractUrlConnection extends DefaultObjectPipe> { + public static final String ACCEPT_HEADER = "accept"; + public static final String ENCODING_HEADER = "accept-charset"; + public static final String CONTENT_TYPE_HEADER = "content-Type"; + static final String ACCEPT_DEFAULT = "*/*"; + static final String ENCODING_DEFAULT = "UTF-8"; + private static Boolean doOutput = false; + private final Map headers = new HashMap<>(); + private final Pattern headerFieldSeparator = Pattern.compile("\n"); + private final Pattern headerValueSeparator = Pattern.compile(":"); + private URLConnection con; + + /** + * Sets the HTTP {@value #ACCEPT_HEADER} header. This is a mime-type such as text/plain + * or text/html. + * + * @param accept mime-type to use for the HTTP accept header + */ + public void setAccept(final String accept) { + setHeader(ACCEPT_HEADER, accept); + } + + /** + * Sets the preferred encoding of the HTTP response. This value is set as the + * {@value #ENCODING_HEADER} header. Additionally, the encoding is used for reading the + * HTTP response if it does not specify an encoding. + * + * @param encoding name of the encoding used for the accept-charset HTTP + * header + */ + public void setEncoding(final String encoding) { + setHeader(ENCODING_HEADER, encoding); + } + + /** + * Sets the HTTP {@value #CONTENT_TYPE_HEADER} header. This is a mime-type such as text/plain, + * text/html or application/x-ndjson. + * + * @param contentType mime-type to use for the HTTP contentType header + */ + public void setContentType(final String contentType) { + setHeader(CONTENT_TYPE_HEADER, contentType); + } + + /** + * Set the DoOutput flag to true if you intend to use the URL connection for output, false if not. The default is false. + * + * @param doOutput the new value + * @see URLConnection#setDoOutput(boolean) + */ + public void setDoOutput(final boolean doOutput) { + AbstractUrlConnection.doOutput = doOutput; + } + + /** + * Gets the {@value #ENCODING_HEADER} header of the URLConnection. + * + * @return the name of the encoding header + */ + public String getEncodingHeader() { + return ENCODING_HEADER; + } + + /** + * Sets a request property, or multiple request properties separated by + * {@code \n}. + * + * @param header request property line + */ + public void setHeader(final String header) { + Arrays.stream(headerFieldSeparator.split(header)).forEach(h -> { + final String[] parts = headerValueSeparator.split(h, 2); + if (parts.length == 2) { + setHeader(parts[0], parts[1].trim()); + } + else { + throw new IllegalArgumentException("Invalid header: " + h); + } + }); + } + + /** + * Sets a request property. + * + * @param key request property key + * @param value request property value + */ + public void setHeader(final String key, final String value) { + headers.put(key.toLowerCase(), value); + } + + /** + * Gets headers of the {@link java.net.URLConnection} as a HashMap. + * + * @return the headers od the URLConnection as HashMap + */ + public Map getHeaders() { + return headers; + } + + /** + * Opens an {@link java.net.URLConnection} defined by the URL. + * + * @param urlStr an URL as String + * @return {@link java.net.URLConnection} + * @throws IOException if an I/O IOException occurs + */ + public URLConnection getUrlConnection(final String urlStr) throws IOException { + final URL url = new URL(urlStr); + con = url.openConnection(); + con.setDoOutput(doOutput); + headers.forEach(con::addRequestProperty); + return con; + } + +} diff --git a/metafacture-io/src/main/java/org/metafacture/io/HttpOpener.java b/metafacture-io/src/main/java/org/metafacture/io/HttpOpener.java index e69e8865c..f6659b113 100644 --- a/metafacture-io/src/main/java/org/metafacture/io/HttpOpener.java +++ b/metafacture-io/src/main/java/org/metafacture/io/HttpOpener.java @@ -1,5 +1,5 @@ /* - * Copyright 2013, 2014 Deutsche Nationalbibliothek + * Copyright 2013 - 2022 Deutsche Nationalbibliothek et al. * * Licensed under the Apache License, Version 2.0 the "License"; * you may not use this file except in compliance with the License. @@ -18,24 +18,17 @@ import org.metafacture.framework.FluxCommand; import org.metafacture.framework.MetafactureException; -import org.metafacture.framework.ObjectReceiver; import org.metafacture.framework.annotations.Description; import org.metafacture.framework.annotations.In; import org.metafacture.framework.annotations.Out; -import org.metafacture.framework.helpers.DefaultObjectPipe; import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; -import java.net.URL; import java.net.URLConnection; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.regex.Pattern; /** - * Opens a {@link URLConnection} and passes a reader to the receiver. + * Opens a {@link java.net.URLConnection} and passes a reader to the receiver. * * @author Christoph Böhme * @author Jan Schnasse @@ -44,88 +37,25 @@ @In(String.class) @Out(Reader.class) @FluxCommand("open-http") -public final class HttpOpener extends DefaultObjectPipe> { - - private static final Pattern HEADER_FIELD_SEPARATOR = Pattern.compile("\n"); - private static final Pattern HEADER_VALUE_SEPARATOR = Pattern.compile(":"); - - private static final String ACCEPT_HEADER = "accept"; - private static final String ENCODING_HEADER = "accept-charset"; - - private static final String ACCEPT_DEFAULT = "*/*"; - private static final String ENCODING_DEFAULT = "UTF-8"; - - private final Map headers = new HashMap<>(); +public final class HttpOpener extends AbstractUrlConnection { /** * Creates an instance of {@link HttpOpener}. + * Sets the default value for the {@value #ENCODING_HEADER} is {@value #ENCODING_DEFAULT}. + * The default value of the {@value #ACCEPT_HEADER} is */* which mean any mime-type. */ public HttpOpener() { setAccept(ACCEPT_DEFAULT); setEncoding(ENCODING_DEFAULT); } - /** - * Sets the HTTP accept header value. This is a mime-type such as text/plain - * or text/html. The default value of the accept is */* which means - * any mime-type. - * - * @param accept mime-type to use for the HTTP accept header - */ - public void setAccept(final String accept) { - setHeader(ACCEPT_HEADER, accept); - } - - /** - * Sets the preferred encoding of the HTTP response. This value is in the - * accept-charset header. Additonally, the encoding is used for reading the - * HTTP resonse if it does not specify an encoding. The default value for - * the encoding is UTF-8. - * - * @param encoding name of the encoding used for the accept-charset HTTP - * header - */ - public void setEncoding(final String encoding) { - setHeader(ENCODING_HEADER, encoding); - } - - /** - * Sets a request property, or multiple request properties separated by - * {@code \n}. - * - * @param header request property line - */ - public void setHeader(final String header) { - Arrays.stream(HEADER_FIELD_SEPARATOR.split(header)).forEach(h -> { - final String[] parts = HEADER_VALUE_SEPARATOR.split(h, 2); - if (parts.length == 2) { - setHeader(parts[0], parts[1].trim()); - } - else { - throw new IllegalArgumentException("Invalid header: " + h); - } - }); - } - - /** - * Sets a request property. - * - * @param key request property key - * @param value request property value - */ - public void setHeader(final String key, final String value) { - headers.put(key.toLowerCase(), value); - } - @Override public void process(final String urlStr) { try { - final URL url = new URL(urlStr); - final URLConnection con = url.openConnection(); - headers.forEach(con::addRequestProperty); + final URLConnection con = getUrlConnection(urlStr); String enc = con.getContentEncoding(); if (enc == null) { - enc = headers.get(ENCODING_HEADER); + enc = getHeaders().get(ENCODING_HEADER); } getReceiver().process(new InputStreamReader(con.getInputStream(), enc)); } diff --git a/metafacture-io/src/main/java/org/metafacture/io/HttpPoster.java b/metafacture-io/src/main/java/org/metafacture/io/HttpPoster.java new file mode 100644 index 000000000..48a54aeda --- /dev/null +++ b/metafacture-io/src/main/java/org/metafacture/io/HttpPoster.java @@ -0,0 +1,89 @@ +/* + * Copyright 2013, 2014 Deutsche Nationalbibliothek + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.io; + +import org.metafacture.framework.FluxCommand; +import org.metafacture.framework.MetafactureException; +import org.metafacture.framework.annotations.Description; +import org.metafacture.framework.annotations.In; +import org.metafacture.framework.annotations.Out; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.Reader; +import java.net.HttpURLConnection; +import java.net.ProtocolException; + +/** + * Uploads data using {@link java.net.HttpURLConnection} with POST method and passes the response to the receiver. + * Supports the setting of 'Accept', 'ContentType' and 'Encoding' as HTTP header fields. + * + * @author Pascal Christoph (dr0i) + */ +@Description("POSTs data to a {@link java.net.HttpURLConnection}. Argument 'url' is mandatory. Supports the setting of 'accept', 'contentType' and 'encoding' (of the response) as http header fields.") +@In(String.class) +@Out(Reader.class) +@FluxCommand("post-http") +public final class HttpPoster extends AbstractUrlConnection { + + private static final String POST = "POST"; + private static final Boolean DO_OUTPUT = true; + private static final int HTTP_STATUS_CODE_MIN = 100; + private static final int HTTP_STATUS_CODE_MAX = 399; + private String contentType = "application/json"; + private String url; + + /** + * Creates an instance of {@link HttpPoster}. + * + * @throws ProtocolException if a protocol error occurs + */ + public HttpPoster() throws ProtocolException { + setDoOutput(DO_OUTPUT); + } + + /** + * Sets the HTTP URL to POST to + * + * @param url the URL to POST to + */ + public void setUrl(final String url) { + this.url = url; + } + + @Override + public void process(final String data) throws IllegalStateException, NullPointerException { + try { + final HttpURLConnection conn = (HttpURLConnection) getUrlConnection(url); + conn.setRequestMethod(POST); + final InputStreamReader inputStreamReader; + final OutputStream os = conn.getOutputStream(); + os.write(data.getBytes()); + if (HTTP_STATUS_CODE_MIN <= conn.getResponseCode() && conn.getResponseCode() <= HTTP_STATUS_CODE_MAX) { + inputStreamReader = new InputStreamReader(conn.getInputStream()); + } + else { + inputStreamReader = new InputStreamReader(conn.getErrorStream()); + } + getReceiver().process(inputStreamReader); + } + catch (final IOException e) { + throw new MetafactureException(e); + } + } +} diff --git a/metafacture-io/src/main/resources/flux-commands.properties b/metafacture-io/src/main/resources/flux-commands.properties index 39540d47e..a23c8082b 100644 --- a/metafacture-io/src/main/resources/flux-commands.properties +++ b/metafacture-io/src/main/resources/flux-commands.properties @@ -22,3 +22,4 @@ write org.metafacture.io.ObjectWriter as-records org.metafacture.io.RecordReader open-resource org.metafacture.io.ResourceOpener open-tar org.metafacture.io.TarReader +post-http org.metafacture.io.HttpPoster