diff --git a/aws/kinesis/.mvn/wrapper/maven-wrapper.jar b/aws/kinesis/.mvn/wrapper/maven-wrapper.jar new file mode 100644 index 0000000..c1dd12f Binary files /dev/null and b/aws/kinesis/.mvn/wrapper/maven-wrapper.jar differ diff --git a/aws/kinesis/.mvn/wrapper/maven-wrapper.properties b/aws/kinesis/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 0000000..e83fa69 --- /dev/null +++ b/aws/kinesis/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip +wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar diff --git a/aws/kinesis/README.md b/aws/kinesis/README.md new file mode 100644 index 0000000..1379116 --- /dev/null +++ b/aws/kinesis/README.md @@ -0,0 +1,8 @@ +# Getting started with AWS Kinesis + + + +Blog posts about this topic: + +* [Getting started with AWS Kinesis](https://reflectoring.io/getting-started-with-aws-kinesis/) + diff --git a/aws/kinesis/mvnw b/aws/kinesis/mvnw new file mode 100755 index 0000000..5643201 --- /dev/null +++ b/aws/kinesis/mvnw @@ -0,0 +1,316 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /usr/local/etc/mavenrc ] ; then + . /usr/local/etc/mavenrc + fi + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "`uname`" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + export JAVA_HOME="`/usr/libexec/java_home`" + else + export JAVA_HOME="/Library/Java/Home" + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +if [ -z "$M2_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + M2_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + M2_HOME=`cd "$M2_HOME" && pwd` + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --unix "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# For Mingw, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$M2_HOME" ] && + M2_HOME="`(cd "$M2_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + if $darwin ; then + javaHome="`dirname \"$javaExecutable\"`" + javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" + else + javaExecutable="`readlink -f \"$javaExecutable\"`" + fi + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`\\unset -f command; \\command -v java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=`cd "$wdir/.."; pwd` + fi + # end of workaround + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' < "$1")" + fi +} + +BASE_DIR=`find_maven_basedir "$(pwd)"` +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +########################################################################################## +# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +# This allows using the maven wrapper in projects that prohibit checking in binary data. +########################################################################################## +if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found .mvn/wrapper/maven-wrapper.jar" + fi +else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." + fi + if [ -n "$MVNW_REPOURL" ]; then + jarUrl="$MVNW_REPOURL/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar" + else + jarUrl="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar" + fi + while IFS="=" read key value; do + case "$key" in (wrapperUrl) jarUrl="$value"; break ;; + esac + done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" + if [ "$MVNW_VERBOSE" = true ]; then + echo "Downloading from: $jarUrl" + fi + wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" + if $cygwin; then + wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"` + fi + + if command -v wget > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found wget ... using wget" + fi + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + wget "$jarUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" + else + wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" + fi + elif command -v curl > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found curl ... using curl" + fi + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + curl -o "$wrapperJarPath" "$jarUrl" -f + else + curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f + fi + + else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Falling back to using Java to download" + fi + javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" + # For Cygwin, switch paths to Windows format before running javac + if $cygwin; then + javaClass=`cygpath --path --windows "$javaClass"` + fi + if [ -e "$javaClass" ]; then + if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Compiling MavenWrapperDownloader.java ..." + fi + # Compiling the Java class + ("$JAVA_HOME/bin/javac" "$javaClass") + fi + if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + # Running the downloader + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Running MavenWrapperDownloader.java ..." + fi + ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") + fi + fi + fi +fi +########################################################################################## +# End of extension +########################################################################################## + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} +if [ "$MVNW_VERBOSE" = true ]; then + echo $MAVEN_PROJECTBASEDIR +fi +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --path --windows "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` +fi + +# Provide a "standardized" way to retrieve the CLI args that will +# work with both Windows and non-Windows executions. +MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" +export MAVEN_CMD_LINE_ARGS + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +exec "$JAVACMD" \ + $MAVEN_OPTS \ + $MAVEN_DEBUG_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" \ + "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/aws/kinesis/mvnw.cmd b/aws/kinesis/mvnw.cmd new file mode 100644 index 0000000..23b7079 --- /dev/null +++ b/aws/kinesis/mvnw.cmd @@ -0,0 +1,188 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Maven Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM set title of command window +title %0 +@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%USERPROFILE%\mavenrc_pre.bat" call "%USERPROFILE%\mavenrc_pre.bat" %* +if exist "%USERPROFILE%\mavenrc_pre.cmd" call "%USERPROFILE%\mavenrc_pre.cmd" %* +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" +set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar" + +FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( + IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B +) + +@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +@REM This allows using the maven wrapper in projects that prohibit checking in binary data. +if exist %WRAPPER_JAR% ( + if "%MVNW_VERBOSE%" == "true" ( + echo Found %WRAPPER_JAR% + ) +) else ( + if not "%MVNW_REPOURL%" == "" ( + SET DOWNLOAD_URL="%MVNW_REPOURL%/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar" + ) + if "%MVNW_VERBOSE%" == "true" ( + echo Couldn't find %WRAPPER_JAR%, downloading it ... + echo Downloading from: %DOWNLOAD_URL% + ) + + powershell -Command "&{"^ + "$webclient = new-object System.Net.WebClient;"^ + "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ + "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ + "}"^ + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^ + "}" + if "%MVNW_VERBOSE%" == "true" ( + echo Finished downloading %WRAPPER_JAR% + ) +) +@REM End of extension + +@REM Provide a "standardized" way to retrieve the CLI args that will +@REM work with both Windows and non-Windows executions. +set MAVEN_CMD_LINE_ARGS=%* + +%MAVEN_JAVA_EXE% ^ + %JVM_CONFIG_MAVEN_PROPS% ^ + %MAVEN_OPTS% ^ + %MAVEN_DEBUG_OPTS% ^ + -classpath %WRAPPER_JAR% ^ + "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" ^ + %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%"=="" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%USERPROFILE%\mavenrc_post.bat" call "%USERPROFILE%\mavenrc_post.bat" +if exist "%USERPROFILE%\mavenrc_post.cmd" call "%USERPROFILE%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%"=="on" pause + +if "%MAVEN_TERMINATE_CMD%"=="on" exit %ERROR_CODE% + +cmd /C exit /B %ERROR_CODE% diff --git a/aws/kinesis/pom.xml b/aws/kinesis/pom.xml new file mode 100644 index 0000000..245aea8 --- /dev/null +++ b/aws/kinesis/pom.xml @@ -0,0 +1,72 @@ + + 4.0.0 + io.pratik + kinesisexamples + 0.0.1-SNAPSHOT + + 1.8 + 1.8 + + + + + software.amazon.awssdk + kinesis + + + + software.amazon.awssdk + firehose + + + software.amazon.awssdk + kinesisanalyticsv2 + + + software.amazon.awssdk + kinesisvideo + + + software.amazon.awssdk + kinesisvideomedia + + + com.amazonaws + amazon-kinesis-video-streams-producer-sdk-java + 1.12.0 + + + + org.apache.flink + flink-streaming-java_2.11 + 1.14.3 + provided + + + org.apache.flink + flink-clients_2.11 + 1.14.3 + + + software.amazon.kinesis + amazon-kinesis-connector-flink + 2.3.0 + + + + + + software.amazon.awssdk + bom + 2.17.116 + pom + import + + + + \ No newline at end of file diff --git a/aws/kinesis/src/main/java/io/pratik/Constants.java b/aws/kinesis/src/main/java/io/pratik/Constants.java new file mode 100644 index 0000000..3b7768b --- /dev/null +++ b/aws/kinesis/src/main/java/io/pratik/Constants.java @@ -0,0 +1,16 @@ +/** + * + */ +package io.pratik; + +import software.amazon.awssdk.regions.Region; + +/** + * @author pratikdas + * + */ +public interface Constants { + String MY_DATA_STREAM = "mydatastream"; + String AWS_PROFILE_NAME = "tompoc"; + Region AWS_REGION = Region.AP_SOUTHEAST_2; +} diff --git a/aws/kinesis/src/main/java/io/pratik/DataStreamResourceHelper.java b/aws/kinesis/src/main/java/io/pratik/DataStreamResourceHelper.java new file mode 100644 index 0000000..fbfb08f --- /dev/null +++ b/aws/kinesis/src/main/java/io/pratik/DataStreamResourceHelper.java @@ -0,0 +1,79 @@ +/** + * + */ +package io.pratik; + +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.kinesis.KinesisClient; +import software.amazon.awssdk.services.kinesis.model.CreateStreamRequest; +import software.amazon.awssdk.services.kinesis.model.CreateStreamResponse; +import software.amazon.awssdk.services.kinesis.model.DescribeStreamSummaryRequest; +import software.amazon.awssdk.services.kinesis.model.DescribeStreamSummaryResponse; +import software.amazon.awssdk.services.kinesis.model.ResourceNotFoundException; +import software.amazon.awssdk.services.kinesis.model.StreamDescriptionSummary; +import software.amazon.awssdk.services.kinesis.model.StreamMode; +import software.amazon.awssdk.services.kinesis.model.StreamModeDetails; +import software.amazon.awssdk.services.kinesis.model.StreamStatus; + +/** + * @author pratikdas + * + */ +public class DataStreamResourceHelper { + + + + /** + * @param args + */ + public static void main(String[] args) { + createDataStream(); + + } + + + public static void createDataStream() { + KinesisClient kinesisClient = getKinesisClient(); + + CreateStreamRequest createStreamRequest = CreateStreamRequest.builder().streamName(Constants.MY_DATA_STREAM).streamModeDetails(StreamModeDetails.builder().streamMode(StreamMode.ON_DEMAND).build()).build(); + CreateStreamResponse createStreamResponse = kinesisClient.createStream(createStreamRequest); + + DescribeStreamSummaryRequest describeStreamSummaryRequest = DescribeStreamSummaryRequest.builder().streamName(Constants.MY_DATA_STREAM ).build(); + DescribeStreamSummaryResponse describeStreamSummaryResponse = kinesisClient.describeStreamSummary(describeStreamSummaryRequest ); + + + long startTime = System.currentTimeMillis(); + long endTime = startTime + ( 10 * 60 * 1000 ); + while ( System.currentTimeMillis() < endTime ) { + try { + Thread.sleep(20 * 1000); + } + catch ( Exception e ) {} + + try { + StreamDescriptionSummary streamDescSumm = describeStreamSummaryResponse.streamDescriptionSummary(); + + if(streamDescSumm.streamStatus().equals(StreamStatus.ACTIVE)) break; + try { + Thread.sleep( 1000 ); + }catch ( Exception e ) {} + }catch ( ResourceNotFoundException e ) {} + + + } + + } + + private static KinesisClient getKinesisClient() { + AwsCredentialsProvider credentialsProvider = ProfileCredentialsProvider.create(Constants.AWS_PROFILE_NAME); + + KinesisClient kinesisClient = KinesisClient + .builder() + .credentialsProvider(credentialsProvider) + .region(Region.US_EAST_1).build(); + return kinesisClient; + } + +} diff --git a/aws/kinesis/src/main/java/io/pratik/EventConsumer.java b/aws/kinesis/src/main/java/io/pratik/EventConsumer.java new file mode 100644 index 0000000..a12de61 --- /dev/null +++ b/aws/kinesis/src/main/java/io/pratik/EventConsumer.java @@ -0,0 +1,79 @@ +/** + * + */ +package io.pratik; + +import java.util.List; +import java.util.logging.Logger; + +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.kinesis.KinesisClient; +import software.amazon.awssdk.services.kinesis.model.GetRecordsRequest; +import software.amazon.awssdk.services.kinesis.model.GetRecordsResponse; +import software.amazon.awssdk.services.kinesis.model.GetShardIteratorRequest; +import software.amazon.awssdk.services.kinesis.model.GetShardIteratorResponse; +import software.amazon.awssdk.services.kinesis.model.Record; +import software.amazon.awssdk.services.kinesis.model.ShardIteratorType; + +/** + * @author pratikdas + * + */ +public class EventConsumer { + private static final Logger logger = Logger.getLogger(EventConsumer.class.getName()); + + /** + * @param args + */ + public static void main(String[] args) { + receiveEvents(); + + } + + public static void receiveEventsWithKCL() {} + + + + public static void receiveEvents() { + KinesisClient kinesisClient = getKinesisClient(); + + String shardId = "shardId-000000000001"; + + GetShardIteratorRequest getShardIteratorRequest = GetShardIteratorRequest.builder().streamName(Constants.MY_DATA_STREAM).shardId(shardId).shardIteratorType(ShardIteratorType.TRIM_HORIZON.name()).build(); + GetShardIteratorResponse getShardIteratorResponse = kinesisClient.getShardIterator(getShardIteratorRequest ); + String shardIterator = getShardIteratorResponse.shardIterator(); + logger.info("shardIterator " + shardIterator); + + while(shardIterator != null) { + GetRecordsRequest getRecordsRequest = GetRecordsRequest.builder().shardIterator(shardIterator).limit(5).build(); + GetRecordsResponse getRecordsResponse = kinesisClient.getRecords(getRecordsRequest ); + + + List records = getRecordsResponse.records(); + + logger.info("count "+records.size()); + records.forEach(record->{ + byte[] dataInBytes = record.data().asByteArray(); + logger.info(new String(dataInBytes)); + }); + + shardIterator = getRecordsResponse.nextShardIterator(); + } + + kinesisClient.close(); + } + + private static KinesisClient getKinesisClient() { + AwsCredentialsProvider credentialsProvider = ProfileCredentialsProvider.create(Constants.AWS_PROFILE_NAME); + + KinesisClient kinesisClient = KinesisClient + .builder() + .credentialsProvider(credentialsProvider) + .region(Region.US_EAST_1) + .build(); + return kinesisClient; + } + +} diff --git a/aws/kinesis/src/main/java/io/pratik/EventSender.java b/aws/kinesis/src/main/java/io/pratik/EventSender.java new file mode 100644 index 0000000..cbc3753 --- /dev/null +++ b/aws/kinesis/src/main/java/io/pratik/EventSender.java @@ -0,0 +1,104 @@ +/** + * + */ +package io.pratik; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Logger; + +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.kinesis.KinesisClient; +import software.amazon.awssdk.services.kinesis.model.PutRecordRequest; +import software.amazon.awssdk.services.kinesis.model.PutRecordResponse; +import software.amazon.awssdk.services.kinesis.model.PutRecordsRequest; +import software.amazon.awssdk.services.kinesis.model.PutRecordsRequestEntry; +import software.amazon.awssdk.services.kinesis.model.PutRecordsResponse; + +/** + * @author pratikdas + * + */ +public class EventSender { + + private static final Logger logger = Logger.getLogger(EventSender.class.getName()); + + /** + * @param args + */ + public static void main(String[] args) { + sendEvent(); + + } + + public static void sendEvent() { + KinesisClient kinesisClient = getKinesisClient(); + + String partitionKey = String.format("partitionKey-%d", 1); + SdkBytes data = SdkBytes.fromByteBuffer(ByteBuffer.wrap("Test data".getBytes())); + PutRecordRequest putRecordRequest + = PutRecordRequest + .builder() + .streamName(Constants.MY_DATA_STREAM) + .partitionKey(partitionKey) + .data(data) + .build(); + + PutRecordResponse putRecordResult + = kinesisClient.putRecord(putRecordRequest); + + logger.info("Put Result" + putRecordResult); + kinesisClient.close(); + } + + public static void sendEvents() { + KinesisClient kinesisClient = getKinesisClient(); + + String partitionKey = String.format("partitionKey-%d", 1); + + + List putRecordsRequestEntryList = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + SdkBytes data = SdkBytes + .fromByteBuffer(ByteBuffer.wrap(("Test event "+i).getBytes())); + + PutRecordsRequestEntry putRecordsRequestEntry + = PutRecordsRequestEntry.builder() + + .data(data) + .partitionKey(partitionKey) + .build(); + + putRecordsRequestEntryList.add(putRecordsRequestEntry); + } + + + PutRecordsRequest putRecordsRequest + = PutRecordsRequest + .builder() + .streamName(Constants.MY_DATA_STREAM) + .records(putRecordsRequestEntryList) + .build(); + + PutRecordsResponse putRecordsResult = kinesisClient + .putRecords(putRecordsRequest); + + logger.info("Put Result" + putRecordsResult); + kinesisClient.close(); + } + + private static KinesisClient getKinesisClient() { + AwsCredentialsProvider credentialsProvider = ProfileCredentialsProvider.create(Constants.AWS_PROFILE_NAME); + + KinesisClient kinesisClient = KinesisClient + .builder() + .credentialsProvider(credentialsProvider) + .region(Region.US_EAST_1).build(); + return kinesisClient; + } + +} diff --git a/aws/kinesis/src/main/java/io/pratik/FirehoseEventSender.java b/aws/kinesis/src/main/java/io/pratik/FirehoseEventSender.java new file mode 100644 index 0000000..dcef4bc --- /dev/null +++ b/aws/kinesis/src/main/java/io/pratik/FirehoseEventSender.java @@ -0,0 +1,63 @@ +/** + * + */ +package io.pratik; + +import java.nio.ByteBuffer; +import java.util.logging.Logger; + +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.firehose.FirehoseClient; +import software.amazon.awssdk.services.firehose.model.PutRecordRequest; +import software.amazon.awssdk.services.firehose.model.PutRecordResponse; +import software.amazon.awssdk.services.firehose.model.Record; + +/** + * @author pratikdas + * + */ +public class FirehoseEventSender { + private final static Logger logger = Logger.getLogger(FirehoseEventSender.class.getName()); + + /** + * @param args + */ + public static void main(String[] args) { + new FirehoseEventSender().sendEvent(); + + } + + public void sendEvent() { + String deliveryStreamName= "PUT-S3-5ZGgA"; + + String data = "Test data" + "\n"; + ; + Record record = Record.builder().data(SdkBytes.fromByteArray(data.getBytes())).build(); + + PutRecordRequest putRecordRequest = PutRecordRequest + .builder() + .deliveryStreamName(deliveryStreamName).record(record).build(); + + FirehoseClient firehoseClient = getFirehoseClient(); + // Put record into the DeliveryStream + PutRecordResponse putRecordResponse = firehoseClient.putRecord(putRecordRequest); + + logger.info("record ID:: " + putRecordResponse.recordId()); + firehoseClient.close(); + } + + private static FirehoseClient getFirehoseClient() { + AwsCredentialsProvider credentialsProvider = ProfileCredentialsProvider.create(Constants.AWS_PROFILE_NAME); + + FirehoseClient kinesisClient = FirehoseClient + .builder() + .credentialsProvider(credentialsProvider) + .region(Constants.AWS_REGION).build(); + return kinesisClient; + } + + +} diff --git a/aws/kinesis/src/main/java/io/pratik/FirehoseResourceHelper.java b/aws/kinesis/src/main/java/io/pratik/FirehoseResourceHelper.java new file mode 100644 index 0000000..72e2807 --- /dev/null +++ b/aws/kinesis/src/main/java/io/pratik/FirehoseResourceHelper.java @@ -0,0 +1,84 @@ +/** + * + */ +package io.pratik; + +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.firehose.FirehoseClient; +import software.amazon.awssdk.services.firehose.model.CreateDeliveryStreamRequest; +import software.amazon.awssdk.services.firehose.model.CreateDeliveryStreamResponse; +import software.amazon.awssdk.services.firehose.model.DeliveryStreamType; +import software.amazon.awssdk.services.firehose.model.ExtendedS3DestinationConfiguration; +import software.amazon.awssdk.services.firehose.model.KinesisStreamSourceConfiguration; +import software.amazon.awssdk.services.firehose.model.S3DestinationConfiguration; +import software.amazon.awssdk.services.kinesis.KinesisClient; +import software.amazon.awssdk.services.kinesis.model.CreateStreamRequest; +import software.amazon.awssdk.services.kinesis.model.CreateStreamResponse; +import software.amazon.awssdk.services.kinesis.model.DescribeStreamSummaryRequest; +import software.amazon.awssdk.services.kinesis.model.DescribeStreamSummaryResponse; +import software.amazon.awssdk.services.kinesis.model.ResourceNotFoundException; +import software.amazon.awssdk.services.kinesis.model.StreamDescriptionSummary; +import software.amazon.awssdk.services.kinesis.model.StreamMode; +import software.amazon.awssdk.services.kinesis.model.StreamModeDetails; +import software.amazon.awssdk.services.kinesis.model.StreamStatus; + +/** + * @author pratikdas + * + */ +public class FirehoseResourceHelper { + + /** + * @param args + */ + public static void main(String[] args) { + createDeliveryStream(); + + } + + + public static void createDeliveryStream() { + FirehoseClient firehoseClient = getFirehoseClient(); + + String kinesisStreamARN = ""; + + String roleARN = ""; + KinesisStreamSourceConfiguration kinesisStreamSourceConfiguration = + KinesisStreamSourceConfiguration.builder().build(); + + String bucketARN = ""; + ExtendedS3DestinationConfiguration s3DestinationConfiguration + = ExtendedS3DestinationConfiguration.builder() + .bucketARN(bucketARN) + .build(); + + String streamName = ""; + CreateDeliveryStreamRequest createDeliveryStreamRequest = + CreateDeliveryStreamRequest + .builder() + .deliveryStreamName(streamName ) + .deliveryStreamType(DeliveryStreamType.DIRECT_PUT) + .kinesisStreamSourceConfiguration(kinesisStreamSourceConfiguration ) + .extendedS3DestinationConfiguration(s3DestinationConfiguration ) + .build(); + CreateDeliveryStreamResponse response = firehoseClient.createDeliveryStream(createDeliveryStreamRequest ); + firehoseClient.close(); + } + + private static FirehoseClient getFirehoseClient() { + AwsCredentialsProvider credentialsProvider = + ProfileCredentialsProvider.create(Constants.AWS_PROFILE_NAME); + + FirehoseClient firehoseClient = + FirehoseClient.builder() + .credentialsProvider(credentialsProvider) + .region(Region.US_EAST_1) + .build(); + + + return firehoseClient; + } + +} diff --git a/aws/kinesis/src/main/java/io/pratik/StreamingDataGenerator.java b/aws/kinesis/src/main/java/io/pratik/StreamingDataGenerator.java new file mode 100644 index 0000000..d879988 --- /dev/null +++ b/aws/kinesis/src/main/java/io/pratik/StreamingDataGenerator.java @@ -0,0 +1,64 @@ +/** + * + */ +package io.pratik; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.net.URL; + +/** + * @author pratikdas + * + */ +public class StreamingDataGenerator { + + /** + * @param args + */ + public static void main(String[] args) { + try { + new StreamingDataGenerator().generate(); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + } + + private void generate() throws Exception { + File file = getFile("apache_access_log"); + + // Note: Double backquote is to avoid compiler + // interpret words + // like \test as \t (ie. as a escape sequence) + + // Creating an object of BufferedReader class + BufferedReader br + = new BufferedReader(new FileReader(file)); + + // Declaring a string variable + String st; + // Condition holds true till + // there is character in a string + while ((st = br.readLine()) != null) { + System.out.println(st); + break; + } + } + + private File getFile(String fileName) throws IOException + { + ClassLoader classLoader = getClass().getClassLoader(); + URL resource = classLoader.getResource(fileName); + + if (resource == null) { + throw new IllegalArgumentException("file is not found!"); + } else { + return new File(resource.getFile()); + } + } + +} diff --git a/aws/kinesis/src/main/java/io/pratik/flink/ErrorCounter.java b/aws/kinesis/src/main/java/io/pratik/flink/ErrorCounter.java new file mode 100644 index 0000000..9415c4b --- /dev/null +++ b/aws/kinesis/src/main/java/io/pratik/flink/ErrorCounter.java @@ -0,0 +1,120 @@ +package io.pratik.flink; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.util.Properties; +import java.util.logging.Logger; + +import org.apache.flink.api.common.functions.FilterFunction; +import org.apache.flink.api.common.functions.FlatMapFunction; +import org.apache.flink.api.common.serialization.SimpleStringSchema; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.util.Collector; + +import io.pratik.Constants; +import io.pratik.models.LogRecord; +import software.amazon.kinesis.connectors.flink.FlinkKinesisProducer; +import software.amazon.kinesis.connectors.flink.config.ConsumerConfigConstants; + +public class ErrorCounter { + private final static Logger logger = Logger.getLogger(ErrorCounter.class.getName()); + private static final String FILE_PATH = "/Users/pratikdas/eclipse-workspace/kinesisexamples/src/main/resources/apache_access_log"; + + public static void main(String[] args) throws Exception { + // set up the streaming execution environment + final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + + DataStream inputStream = createSource(env); + + DataStream logRecords = inputStream.flatMap(new FlatMapFunction() { + + @Override + public void flatMap(String value, Collector out) throws Exception { + + String[] parts = value.split("\\s+"); + + LogRecord record = new LogRecord(); + record.setIp(parts[0]); + record.setHttpStatus(parts[8]); + record.setUrl(parts[6]); + + out.collect(record); + + } + + }); + + DataStream errorRecords = logRecords.filter(new FilterFunction() { + + @Override + public boolean filter(LogRecord value) throws Exception { + boolean matched = !value.getHttpStatus().equalsIgnoreCase("200"); + + return matched; + } + }); + + DataStream keyedStream = errorRecords.keyBy(value -> value.getIp()).flatMap(new FlatMapFunction() { + + @Override + public void flatMap(LogRecord value, Collector out) throws Exception { + out.collect(value.getUrl()+"::" + value.getHttpStatus()); + } + }); + + //TODO Uncomment this code for deploying to Kinesis Data Analytics + + // keyedStream.addSink(createSink()); + + keyedStream.print(); + + env.execute("Error alerts"); + + } + + /*private static void createSink(final StreamExecutionEnvironment env, DataStream input) { + input.print(); + }*/ + + private static DataStream createSource(final StreamExecutionEnvironment env) { + return env.readTextFile( + FILE_PATH); + } + + //TODO Uncomment this code for deploying to Kinesis Data Analytics + /*private static DataStream createSource(final StreamExecutionEnvironment env) { + Properties inputProperties = new Properties(); + inputProperties.setProperty(ConsumerConfigConstants.AWS_REGION, Constants.AWS_REGION.toString()); + inputProperties.setProperty(ConsumerConfigConstants.STREAM_INITIAL_POSITION, "LATEST"); + + String inputStreamName = "in-app-log-stream"; + return env.addSource(new FlinkKinesisConsumer<>(inputStreamName, new SimpleStringSchema(), inputProperties)); + }*/ + + private static FlinkKinesisProducer createSink() { + Properties outputProperties = new Properties(); + outputProperties.setProperty(ConsumerConfigConstants.AWS_REGION, Constants.AWS_REGION.toString()); + + FlinkKinesisProducer sink = new FlinkKinesisProducer<>(new SimpleStringSchema(), outputProperties); + String outputStreamName = "log_data_stream"; + sink.setDefaultStream(outputStreamName); + sink.setDefaultPartition("0"); + + return sink; + } + + + + private File getFile(String fileName) throws IOException { + ClassLoader classLoader = getClass().getClassLoader(); + URL resource = classLoader.getResource(fileName); + + if (resource == null) { + throw new IllegalArgumentException("file is not found!"); + } else { + return new File(resource.getFile()); + } + } +} diff --git a/aws/kinesis/src/main/java/io/pratik/models/LogRecord.java b/aws/kinesis/src/main/java/io/pratik/models/LogRecord.java new file mode 100644 index 0000000..560d891 --- /dev/null +++ b/aws/kinesis/src/main/java/io/pratik/models/LogRecord.java @@ -0,0 +1,40 @@ +/** + * + */ +package io.pratik.models; + +/** + * @author pratikdas + * + */ +public class LogRecord { + private String ip; + private String httpStatus; + private String url; + public String getIp() { + return ip; + } + public void setIp(String ip) { + this.ip = ip; + } + public String getHttpStatus() { + return httpStatus; + } + public void setHttpStatus(String httpStatus) { + this.httpStatus = httpStatus; + } + public String getUrl() { + return url; + } + public void setUrl(String url) { + this.url = url; + } + @Override + public String toString() { + return "LogRecord [ip=" + ip + ", httpStatus=" + httpStatus + ", url=" + url + "]"; + } + + + + +} diff --git a/aws/kinesis/src/main/resources/apache_access_log b/aws/kinesis/src/main/resources/apache_access_log new file mode 100644 index 0000000..3d8d626 --- /dev/null +++ b/aws/kinesis/src/main/resources/apache_access_log @@ -0,0 +1,33 @@ +83.149.9.216 - - [17/May/2015:10:05:03 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-search.png HTTP/1.1" 200 203023 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" +83.149.9.216 - - [17/May/2015:10:05:43 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-dashboard3.png HTTP/1.1" 200 171717 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" +83.149.9.216 - - [17/May/2015:10:05:47 +0000] "GET /presentations/logstash-monitorama-2013/plugin/highlight/highlight.js HTTP/1.1" 200 26185 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" +83.149.9.216 - - [17/May/2015:10:05:12 +0000] "GET /presentations/logstash-monitorama-2013/plugin/zoom-js/zoom.js HTTP/1.1" 200 7697 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" +83.149.9.216 - - [17/May/2015:10:05:07 +0000] "GET /presentations/logstash-monitorama-2013/plugin/notes/notes.js HTTP/1.1" 200 2892 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" +83.149.9.216 - - [17/May/2015:10:05:34 +0000] "GET /presentations/logstash-monitorama-2013/images/sad-medic.png HTTP/1.1" 200 430406 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" +83.149.9.216 - - [17/May/2015:10:05:57 +0000] "GET /presentations/logstash-monitorama-2013/css/fonts/Roboto-Bold.ttf HTTP/1.1" 404 38720 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" +83.149.9.216 - - [17/May/2015:10:05:50 +0000] "GET /presentations/logstash-monitorama-2013/css/fonts/Roboto-Regular.ttf HTTP/1.1" 200 41820 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" +83.149.9.216 - - [17/May/2015:10:05:24 +0000] "GET /presentations/logstash-monitorama-2013/images/frontend-response-codes.png HTTP/1.1" 310 52878 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36" +46.105.14.53 - - [17/May/2015:10:05:44 +0000] "GET /blog/tags/puppet?flav=rss20 HTTP/1.1" 200 14872 "-" "UniversalFeedParser/4.2-pre-314-svn +http://feedparser.org/" +20.241.23.22 - - [17/May/2015:10:05:47 +0000] "GET /blog/geekery/soekris-gpio.html HTTP/1.0" 200 9587 "http://www.semicomplete.com/blog/tags/C" "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)" +91.177.205.119 - - [17/May/2015:10:05:22 +0000] "GET /blog/geekery/xvfb-firefox.html HTTP/1.1" 200 10975 "http://en.wikipedia.org/wiki/Xvfb" "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)" +91.177.205.119 - - [17/May/2015:10:05:34 +0000] "GET /reset.css HTTP/1.1" 200 1015 "http://semicomplete.com/blog/geekery/xvfb-firefox.html" "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)" +91.177.205.119 - - [17/May/2015:10:05:37 +0000] "GET /style2.css HTTP/1.1" 200 4877 "http://semicomplete.com/blog/geekery/xvfb-firefox.html" "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)" +91.177.205.119 - - [17/May/2015:10:05:54 +0000] "GET /images/jordan-80.png HTTP/1.1" 200 6146 "http://semicomplete.com/blog/geekery/xvfb-firefox.html" "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)" +91.177.205.119 - - [17/May/2015:10:05:31 +0000] "GET /images/web/2009/banner.png HTTP/1.1" 200 52315 "http://semicomplete.com/blog/geekery/xvfb-firefox.html" "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)" +91.177.205.119 - - [17/May/2015:10:05:32 +0000] "GET /favicon.ico HTTP/1.1" 200 3638 "-" "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Win64; x64; Trident/6.0)" +66.249.73.185 - - [17/May/2015:10:05:22 +0000] "GET /doc/index.html?org/elasticsearch/action/search/SearchResponse.html HTTP/1.1" 404 294 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" +207.241.237.228 - - [17/May/2015:10:05:40 +0000] "GET /blog/tags/defcon HTTP/1.0" 200 24142 "http://www.semicomplete.com/blog/tags/C" "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)" +207.241.237.101 - - [17/May/2015:10:05:51 +0000] "GET /blog/tags/regex HTTP/1.0" 200 14888 "http://www.semicomplete.com/blog/tags/C" "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)" +87.169.99.232 - - [17/May/2015:10:05:59 +0000] "GET /presentations/puppet-at-loggly/puppet-at-loggly.pdf.html HTTP/1.1" 200 24747 "https://www.google.de/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36" +209.85.238.199 - - [17/May/2015:10:05:30 +0000] "GET /blog/tags/firefox?flav=rss20 HTTP/1.1" 200 16021 "-" "Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 3 subscribers; feed-id=14171215010336145331)" +209.85.238.199 - - [17/May/2015:10:05:15 +0000] "GET /test.xml HTTP/1.1" 200 1370 "-" "Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 1 subscribers; feed-id=11390274670024826467)" +81.220.24.207 - - [17/May/2015:10:05:13 +0000] "GET /blog/geekery/ssl-latency.html HTTP/1.1" 200 17147 "http://www.google.fr/url?sa=t&rct=j&q=&esrc=s&source=web&cd=5&ved=0CE4QFjAE&url=http%3A%2F%2Fwww.semicomplete.com%2Fblog%2Fgeekery%2Fssl-latency.html&ei=ZdEAU9mGGuWX1AW09IDoBw&usg=AFQjCNHw6zioJpizqX8Q0YpKKaF4zdCSEg&bvm=bv.61535280,d.d2k" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/7.0.1 Safari/537.73.11" +81.220.24.207 - - [17/May/2015:10:05:44 +0000] "GET /reset.css HTTP/1.1" 200 1015 "http://www.semicomplete.com/blog/geekery/ssl-latency.html" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/7.0.1 Safari/537.73.11" +81.220.24.207 - - [17/May/2015:10:05:26 +0000] "GET /images/jordan-80.png HTTP/1.1" 200 6146 "http://www.semicomplete.com/blog/geekery/ssl-latency.html" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/7.0.1 Safari/537.73.11" +81.220.24.207 - - [17/May/2015:10:05:39 +0000] "GET /style2.css HTTP/1.1" 200 4877 "http://www.semicomplete.com/blog/geekery/ssl-latency.html" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/7.0.1 Safari/537.73.11" +81.220.24.207 - - [17/May/2015:10:05:52 +0000] "GET /images/web/2009/banner.png HTTP/1.1" 200 52315 "http://www.semicomplete.com/blog/geekery/ssl-latency.html" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/7.0.1 Safari/537.73.11" +81.220.24.207 - - [17/May/2015:10:05:21 +0000] "GET /favicon.ico HTTP/1.1" 200 3638 "http://www.semicomplete.com/blog/geekery/ssl-latency.html" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/7.0.1 Safari/537.73.11" +66.249.73.135 - - [17/May/2015:11:05:17 +0000] "GET /blog/geekery/vmware-cpu-performance.html HTTP/1.1" 200 12908 "-" "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" +46.105.14.53 - - [17/May/2015:11:05:42 +0000] "GET /blog/tags/puppet?flav=rss20 HTTP/1.1" 200 14872 "-" "UniversalFeedParser/4.2-pre-314-svn +http://feedparser.org/" +218.30.103.62 - - [17/May/2015:11:05:11 +0000] "GET /robots.txt HTTP/1.1" 200 - "-" "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" +218.30.103.62 - - [17/May/2015:11:05:46 +0000] "GET /robots.txt HTTP/1.1" 200 - "-" "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" \ No newline at end of file