Apache Spark Beispiele
Marcus Kessel
parent
db7de9cc8f
commit
18147cd7fe
|
|
@ -53,7 +53,7 @@ public class SecondarySorting {
|
|||
st.nextToken(); // ignore day
|
||||
String temp = st.nextToken();
|
||||
|
||||
System.out.println(new Text(yearMonth + ":" + "000".substring(0, 3-temp.length()) + temp).toString() + " -> " + value);
|
||||
System.out.println("Mapper -> " + new Text(yearMonth + ":" + "000".substring(0, 3-temp.length()) + temp).toString() + " -> " + value);
|
||||
context.write(new Text(yearMonth + ":" + "000".substring(0, 3-temp.length()) + temp), value);
|
||||
}
|
||||
}
|
||||
|
|
@ -84,11 +84,17 @@ public class SecondarySorting {
|
|||
public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
int c = 0;
|
||||
System.out.println("blub");
|
||||
for (Text val : values) {
|
||||
String s = val.toString();
|
||||
System.out.println(s);
|
||||
// note that key changes in each iteration
|
||||
// because of MyGroupingComparator (i.e., groups keys)
|
||||
System.out.println("Reducer -> " + key + " = " + s);
|
||||
sb.append(s.substring(s.lastIndexOf(",") + 1, s.length()).trim() + " ");
|
||||
c++;
|
||||
}
|
||||
System.out.println("Reducer (#values = " + c + ")");
|
||||
|
||||
result.set(sb.toString());
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,9 @@ public class WordCountVL {
|
|||
private Text word = new Text();
|
||||
|
||||
public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
|
||||
//System.out.println(key);
|
||||
//System.out.println(value);
|
||||
|
||||
String[] woerter = value.toString().split("\\W+");
|
||||
|
||||
for (String wort: woerter) {
|
||||
|
|
@ -52,7 +55,7 @@ public class WordCountVL {
|
|||
|
||||
public static class DoNothingReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
|
||||
public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
|
||||
context.write(key, values.iterator().next());
|
||||
context.write(key, values.iterator().next()); // nur ein Wert, siehe IdentityMapper.class
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -67,7 +70,7 @@ public class WordCountVL {
|
|||
job.setMapperClass(TokenizerMapper.class);
|
||||
job.setCombinerClass(IntSumReducer.class); // reduce lokal bei den Mappern
|
||||
job.setReducerClass(IntSumReducer.class); // reduce nach Verteilung bei den Reducern
|
||||
job.setNumReduceTasks(4);
|
||||
job.setNumReduceTasks(4); // 4 Tasks parallel
|
||||
job.setOutputKeyClass(Text.class);
|
||||
job.setOutputValueClass(IntWritable.class);
|
||||
|
||||
|
|
@ -81,11 +84,11 @@ public class WordCountVL {
|
|||
|
||||
// --- Ende Job 1 ---
|
||||
|
||||
job = Job.getInstance(conf, "word count sort");
|
||||
job = Job.getInstance(conf, "word count sort"); // natural ordering of keys
|
||||
job.setJarByClass(WordCountVL.class);
|
||||
job.setMapperClass(IdentityMapper.class);
|
||||
job.setReducerClass(DoNothingReducer.class);
|
||||
job.setNumReduceTasks(1);
|
||||
job.setNumReduceTasks(1); // 1 Task parallel
|
||||
job.setOutputKeyClass(Text.class);
|
||||
job.setOutputValueClass(IntWritable.class);
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,41 @@
|
|||
# ---> Eclipse
|
||||
*.pydevproject
|
||||
.metadata
|
||||
.gradle
|
||||
bin/
|
||||
tmp/
|
||||
*.tmp
|
||||
*.bak
|
||||
*.swp
|
||||
*~.nib
|
||||
local.properties
|
||||
.settings/
|
||||
.loadpath
|
||||
|
||||
# Eclipse Core
|
||||
.project
|
||||
|
||||
# External tool builders
|
||||
.externalToolBuilders/
|
||||
|
||||
# Locally stored "Eclipse launch configurations"
|
||||
*.launch
|
||||
|
||||
# CDT-specific
|
||||
.cproject
|
||||
|
||||
# JDT-specific (Eclipse Java Development Tools)
|
||||
.classpath
|
||||
|
||||
# Java annotation processor (APT)
|
||||
.factorypath
|
||||
|
||||
# PDT-specific
|
||||
.buildpath
|
||||
|
||||
# sbteclipse plugin
|
||||
.target
|
||||
|
||||
# TeXlipse plugin
|
||||
.texlipse
|
||||
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
# Spark-It
|
||||
|
||||
Binary file not shown.
|
|
@ -0,0 +1,18 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.1/apache-maven-3.9.1-bin.zip
|
||||
wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
# VM Options
|
||||
|
||||
Java 17 (see https://stackoverflow.com/questions/73465937/apache-spark-3-3-0-breaks-on-java-17-with-cannot-access-class-sun-nio-ch-direct)
|
||||
|
||||
```shell
|
||||
--add-opens=java.base/java.lang=ALL-UNNAMED
|
||||
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED
|
||||
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
|
||||
--add-opens=java.base/java.io=ALL-UNNAMED
|
||||
--add-opens=java.base/java.net=ALL-UNNAMED
|
||||
--add-opens=java.base/java.nio=ALL-UNNAMED
|
||||
--add-opens=java.base/java.util=ALL-UNNAMED
|
||||
--add-opens=java.base/java.util.concurrent=ALL-UNNAMED
|
||||
--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
|
||||
--add-opens=java.base/sun.nio.ch=ALL-UNNAMED
|
||||
--add-opens=java.base/sun.nio.cs=ALL-UNNAMED
|
||||
--add-opens=java.base/sun.security.action=ALL-UNNAMED
|
||||
--add-opens=java.base/sun.util.calendar=ALL-UNNAMED
|
||||
--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED
|
||||
```
|
||||
|
|
@ -0,0 +1,308 @@
|
|||
#!/bin/sh
|
||||
# ----------------------------------------------------------------------------
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Apache Maven Wrapper startup batch script, version 3.2.0
|
||||
#
|
||||
# Required ENV vars:
|
||||
# ------------------
|
||||
# JAVA_HOME - location of a JDK home dir
|
||||
#
|
||||
# Optional ENV vars
|
||||
# -----------------
|
||||
# MAVEN_OPTS - parameters passed to the Java VM when running Maven
|
||||
# e.g. to debug Maven itself, use
|
||||
# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
|
||||
# MAVEN_SKIP_RC - flag to disable loading of mavenrc files
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
if [ -z "$MAVEN_SKIP_RC" ] ; then
|
||||
|
||||
if [ -f /usr/local/etc/mavenrc ] ; then
|
||||
. /usr/local/etc/mavenrc
|
||||
fi
|
||||
|
||||
if [ -f /etc/mavenrc ] ; then
|
||||
. /etc/mavenrc
|
||||
fi
|
||||
|
||||
if [ -f "$HOME/.mavenrc" ] ; then
|
||||
. "$HOME/.mavenrc"
|
||||
fi
|
||||
|
||||
fi
|
||||
|
||||
# OS specific support. $var _must_ be set to either true or false.
|
||||
cygwin=false;
|
||||
darwin=false;
|
||||
mingw=false
|
||||
case "$(uname)" in
|
||||
CYGWIN*) cygwin=true ;;
|
||||
MINGW*) mingw=true;;
|
||||
Darwin*) darwin=true
|
||||
# Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home
|
||||
# See https://developer.apple.com/library/mac/qa/qa1170/_index.html
|
||||
if [ -z "$JAVA_HOME" ]; then
|
||||
if [ -x "/usr/libexec/java_home" ]; then
|
||||
JAVA_HOME="$(/usr/libexec/java_home)"; export JAVA_HOME
|
||||
else
|
||||
JAVA_HOME="/Library/Java/Home"; export JAVA_HOME
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ -z "$JAVA_HOME" ] ; then
|
||||
if [ -r /etc/gentoo-release ] ; then
|
||||
JAVA_HOME=$(java-config --jre-home)
|
||||
fi
|
||||
fi
|
||||
|
||||
# For Cygwin, ensure paths are in UNIX format before anything is touched
|
||||
if $cygwin ; then
|
||||
[ -n "$JAVA_HOME" ] &&
|
||||
JAVA_HOME=$(cygpath --unix "$JAVA_HOME")
|
||||
[ -n "$CLASSPATH" ] &&
|
||||
CLASSPATH=$(cygpath --path --unix "$CLASSPATH")
|
||||
fi
|
||||
|
||||
# For Mingw, ensure paths are in UNIX format before anything is touched
|
||||
if $mingw ; then
|
||||
[ -n "$JAVA_HOME" ] && [ -d "$JAVA_HOME" ] &&
|
||||
JAVA_HOME="$(cd "$JAVA_HOME" || (echo "cannot cd into $JAVA_HOME."; exit 1); pwd)"
|
||||
fi
|
||||
|
||||
if [ -z "$JAVA_HOME" ]; then
|
||||
javaExecutable="$(which javac)"
|
||||
if [ -n "$javaExecutable" ] && ! [ "$(expr "\"$javaExecutable\"" : '\([^ ]*\)')" = "no" ]; then
|
||||
# readlink(1) is not available as standard on Solaris 10.
|
||||
readLink=$(which readlink)
|
||||
if [ ! "$(expr "$readLink" : '\([^ ]*\)')" = "no" ]; then
|
||||
if $darwin ; then
|
||||
javaHome="$(dirname "\"$javaExecutable\"")"
|
||||
javaExecutable="$(cd "\"$javaHome\"" && pwd -P)/javac"
|
||||
else
|
||||
javaExecutable="$(readlink -f "\"$javaExecutable\"")"
|
||||
fi
|
||||
javaHome="$(dirname "\"$javaExecutable\"")"
|
||||
javaHome=$(expr "$javaHome" : '\(.*\)/bin')
|
||||
JAVA_HOME="$javaHome"
|
||||
export JAVA_HOME
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$JAVACMD" ] ; then
|
||||
if [ -n "$JAVA_HOME" ] ; then
|
||||
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
|
||||
# IBM's JDK on AIX uses strange locations for the executables
|
||||
JAVACMD="$JAVA_HOME/jre/sh/java"
|
||||
else
|
||||
JAVACMD="$JAVA_HOME/bin/java"
|
||||
fi
|
||||
else
|
||||
JAVACMD="$(\unset -f command 2>/dev/null; \command -v java)"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ! -x "$JAVACMD" ] ; then
|
||||
echo "Error: JAVA_HOME is not defined correctly." >&2
|
||||
echo " We cannot execute $JAVACMD" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$JAVA_HOME" ] ; then
|
||||
echo "Warning: JAVA_HOME environment variable is not set."
|
||||
fi
|
||||
|
||||
# traverses directory structure from process work directory to filesystem root
|
||||
# first directory with .mvn subdirectory is considered project base directory
|
||||
find_maven_basedir() {
|
||||
if [ -z "$1" ]
|
||||
then
|
||||
echo "Path not specified to find_maven_basedir"
|
||||
return 1
|
||||
fi
|
||||
|
||||
basedir="$1"
|
||||
wdir="$1"
|
||||
while [ "$wdir" != '/' ] ; do
|
||||
if [ -d "$wdir"/.mvn ] ; then
|
||||
basedir=$wdir
|
||||
break
|
||||
fi
|
||||
# workaround for JBEAP-8937 (on Solaris 10/Sparc)
|
||||
if [ -d "${wdir}" ]; then
|
||||
wdir=$(cd "$wdir/.." || exit 1; pwd)
|
||||
fi
|
||||
# end of workaround
|
||||
done
|
||||
printf '%s' "$(cd "$basedir" || exit 1; pwd)"
|
||||
}
|
||||
|
||||
# concatenates all lines of a file
|
||||
concat_lines() {
|
||||
if [ -f "$1" ]; then
|
||||
# Remove \r in case we run on Windows within Git Bash
|
||||
# and check out the repository with auto CRLF management
|
||||
# enabled. Otherwise, we may read lines that are delimited with
|
||||
# \r\n and produce $'-Xarg\r' rather than -Xarg due to word
|
||||
# splitting rules.
|
||||
tr -s '\r\n' ' ' < "$1"
|
||||
fi
|
||||
}
|
||||
|
||||
log() {
|
||||
if [ "$MVNW_VERBOSE" = true ]; then
|
||||
printf '%s\n' "$1"
|
||||
fi
|
||||
}
|
||||
|
||||
BASE_DIR=$(find_maven_basedir "$(dirname "$0")")
|
||||
if [ -z "$BASE_DIR" ]; then
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}; export MAVEN_PROJECTBASEDIR
|
||||
log "$MAVEN_PROJECTBASEDIR"
|
||||
|
||||
##########################################################################################
|
||||
# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
|
||||
# This allows using the maven wrapper in projects that prohibit checking in binary data.
|
||||
##########################################################################################
|
||||
wrapperJarPath="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar"
|
||||
if [ -r "$wrapperJarPath" ]; then
|
||||
log "Found $wrapperJarPath"
|
||||
else
|
||||
log "Couldn't find $wrapperJarPath, downloading it ..."
|
||||
|
||||
if [ -n "$MVNW_REPOURL" ]; then
|
||||
wrapperUrl="$MVNW_REPOURL/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar"
|
||||
else
|
||||
wrapperUrl="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar"
|
||||
fi
|
||||
while IFS="=" read -r key value; do
|
||||
# Remove '\r' from value to allow usage on windows as IFS does not consider '\r' as a separator ( considers space, tab, new line ('\n'), and custom '=' )
|
||||
safeValue=$(echo "$value" | tr -d '\r')
|
||||
case "$key" in (wrapperUrl) wrapperUrl="$safeValue"; break ;;
|
||||
esac
|
||||
done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties"
|
||||
log "Downloading from: $wrapperUrl"
|
||||
|
||||
if $cygwin; then
|
||||
wrapperJarPath=$(cygpath --path --windows "$wrapperJarPath")
|
||||
fi
|
||||
|
||||
if command -v wget > /dev/null; then
|
||||
log "Found wget ... using wget"
|
||||
[ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--quiet"
|
||||
if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
|
||||
wget $QUIET "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath"
|
||||
else
|
||||
wget $QUIET --http-user="$MVNW_USERNAME" --http-password="$MVNW_PASSWORD" "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath"
|
||||
fi
|
||||
elif command -v curl > /dev/null; then
|
||||
log "Found curl ... using curl"
|
||||
[ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--silent"
|
||||
if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
|
||||
curl $QUIET -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath"
|
||||
else
|
||||
curl $QUIET --user "$MVNW_USERNAME:$MVNW_PASSWORD" -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath"
|
||||
fi
|
||||
else
|
||||
log "Falling back to using Java to download"
|
||||
javaSource="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.java"
|
||||
javaClass="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.class"
|
||||
# For Cygwin, switch paths to Windows format before running javac
|
||||
if $cygwin; then
|
||||
javaSource=$(cygpath --path --windows "$javaSource")
|
||||
javaClass=$(cygpath --path --windows "$javaClass")
|
||||
fi
|
||||
if [ -e "$javaSource" ]; then
|
||||
if [ ! -e "$javaClass" ]; then
|
||||
log " - Compiling MavenWrapperDownloader.java ..."
|
||||
("$JAVA_HOME/bin/javac" "$javaSource")
|
||||
fi
|
||||
if [ -e "$javaClass" ]; then
|
||||
log " - Running MavenWrapperDownloader.java ..."
|
||||
("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$wrapperUrl" "$wrapperJarPath") || rm -f "$wrapperJarPath"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
##########################################################################################
|
||||
# End of extension
|
||||
##########################################################################################
|
||||
|
||||
# If specified, validate the SHA-256 sum of the Maven wrapper jar file
|
||||
wrapperSha256Sum=""
|
||||
while IFS="=" read -r key value; do
|
||||
case "$key" in (wrapperSha256Sum) wrapperSha256Sum=$value; break ;;
|
||||
esac
|
||||
done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties"
|
||||
if [ -n "$wrapperSha256Sum" ]; then
|
||||
wrapperSha256Result=false
|
||||
if command -v sha256sum > /dev/null; then
|
||||
if echo "$wrapperSha256Sum $wrapperJarPath" | sha256sum -c > /dev/null 2>&1; then
|
||||
wrapperSha256Result=true
|
||||
fi
|
||||
elif command -v shasum > /dev/null; then
|
||||
if echo "$wrapperSha256Sum $wrapperJarPath" | shasum -a 256 -c > /dev/null 2>&1; then
|
||||
wrapperSha256Result=true
|
||||
fi
|
||||
else
|
||||
echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available."
|
||||
echo "Please install either command, or disable validation by removing 'wrapperSha256Sum' from your maven-wrapper.properties."
|
||||
exit 1
|
||||
fi
|
||||
if [ $wrapperSha256Result = false ]; then
|
||||
echo "Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised." >&2
|
||||
echo "Investigate or delete $wrapperJarPath to attempt a clean download." >&2
|
||||
echo "If you updated your Maven version, you need to update the specified wrapperSha256Sum property." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS"
|
||||
|
||||
# For Cygwin, switch paths to Windows format before running java
|
||||
if $cygwin; then
|
||||
[ -n "$JAVA_HOME" ] &&
|
||||
JAVA_HOME=$(cygpath --path --windows "$JAVA_HOME")
|
||||
[ -n "$CLASSPATH" ] &&
|
||||
CLASSPATH=$(cygpath --path --windows "$CLASSPATH")
|
||||
[ -n "$MAVEN_PROJECTBASEDIR" ] &&
|
||||
MAVEN_PROJECTBASEDIR=$(cygpath --path --windows "$MAVEN_PROJECTBASEDIR")
|
||||
fi
|
||||
|
||||
# Provide a "standardized" way to retrieve the CLI args that will
|
||||
# work with both Windows and non-Windows executions.
|
||||
MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $*"
|
||||
export MAVEN_CMD_LINE_ARGS
|
||||
|
||||
WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
|
||||
|
||||
# shellcheck disable=SC2086 # safe args
|
||||
exec "$JAVACMD" \
|
||||
$MAVEN_OPTS \
|
||||
$MAVEN_DEBUG_OPTS \
|
||||
-classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \
|
||||
"-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \
|
||||
${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@"
|
||||
|
|
@ -0,0 +1,205 @@
|
|||
@REM ----------------------------------------------------------------------------
|
||||
@REM Licensed to the Apache Software Foundation (ASF) under one
|
||||
@REM or more contributor license agreements. See the NOTICE file
|
||||
@REM distributed with this work for additional information
|
||||
@REM regarding copyright ownership. The ASF licenses this file
|
||||
@REM to you under the Apache License, Version 2.0 (the
|
||||
@REM "License"); you may not use this file except in compliance
|
||||
@REM with the License. You may obtain a copy of the License at
|
||||
@REM
|
||||
@REM http://www.apache.org/licenses/LICENSE-2.0
|
||||
@REM
|
||||
@REM Unless required by applicable law or agreed to in writing,
|
||||
@REM software distributed under the License is distributed on an
|
||||
@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
@REM KIND, either express or implied. See the License for the
|
||||
@REM specific language governing permissions and limitations
|
||||
@REM under the License.
|
||||
@REM ----------------------------------------------------------------------------
|
||||
|
||||
@REM ----------------------------------------------------------------------------
|
||||
@REM Apache Maven Wrapper startup batch script, version 3.2.0
|
||||
@REM
|
||||
@REM Required ENV vars:
|
||||
@REM JAVA_HOME - location of a JDK home dir
|
||||
@REM
|
||||
@REM Optional ENV vars
|
||||
@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands
|
||||
@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending
|
||||
@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven
|
||||
@REM e.g. to debug Maven itself, use
|
||||
@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
|
||||
@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files
|
||||
@REM ----------------------------------------------------------------------------
|
||||
|
||||
@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on'
|
||||
@echo off
|
||||
@REM set title of command window
|
||||
title %0
|
||||
@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on'
|
||||
@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO%
|
||||
|
||||
@REM set %HOME% to equivalent of $HOME
|
||||
if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%")
|
||||
|
||||
@REM Execute a user defined script before this one
|
||||
if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre
|
||||
@REM check for pre script, once with legacy .bat ending and once with .cmd ending
|
||||
if exist "%USERPROFILE%\mavenrc_pre.bat" call "%USERPROFILE%\mavenrc_pre.bat" %*
|
||||
if exist "%USERPROFILE%\mavenrc_pre.cmd" call "%USERPROFILE%\mavenrc_pre.cmd" %*
|
||||
:skipRcPre
|
||||
|
||||
@setlocal
|
||||
|
||||
set ERROR_CODE=0
|
||||
|
||||
@REM To isolate internal variables from possible post scripts, we use another setlocal
|
||||
@setlocal
|
||||
|
||||
@REM ==== START VALIDATION ====
|
||||
if not "%JAVA_HOME%" == "" goto OkJHome
|
||||
|
||||
echo.
|
||||
echo Error: JAVA_HOME not found in your environment. >&2
|
||||
echo Please set the JAVA_HOME variable in your environment to match the >&2
|
||||
echo location of your Java installation. >&2
|
||||
echo.
|
||||
goto error
|
||||
|
||||
:OkJHome
|
||||
if exist "%JAVA_HOME%\bin\java.exe" goto init
|
||||
|
||||
echo.
|
||||
echo Error: JAVA_HOME is set to an invalid directory. >&2
|
||||
echo JAVA_HOME = "%JAVA_HOME%" >&2
|
||||
echo Please set the JAVA_HOME variable in your environment to match the >&2
|
||||
echo location of your Java installation. >&2
|
||||
echo.
|
||||
goto error
|
||||
|
||||
@REM ==== END VALIDATION ====
|
||||
|
||||
:init
|
||||
|
||||
@REM Find the project base dir, i.e. the directory that contains the folder ".mvn".
|
||||
@REM Fallback to current working directory if not found.
|
||||
|
||||
set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR%
|
||||
IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir
|
||||
|
||||
set EXEC_DIR=%CD%
|
||||
set WDIR=%EXEC_DIR%
|
||||
:findBaseDir
|
||||
IF EXIST "%WDIR%"\.mvn goto baseDirFound
|
||||
cd ..
|
||||
IF "%WDIR%"=="%CD%" goto baseDirNotFound
|
||||
set WDIR=%CD%
|
||||
goto findBaseDir
|
||||
|
||||
:baseDirFound
|
||||
set MAVEN_PROJECTBASEDIR=%WDIR%
|
||||
cd "%EXEC_DIR%"
|
||||
goto endDetectBaseDir
|
||||
|
||||
:baseDirNotFound
|
||||
set MAVEN_PROJECTBASEDIR=%EXEC_DIR%
|
||||
cd "%EXEC_DIR%"
|
||||
|
||||
:endDetectBaseDir
|
||||
|
||||
IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig
|
||||
|
||||
@setlocal EnableExtensions EnableDelayedExpansion
|
||||
for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a
|
||||
@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS%
|
||||
|
||||
:endReadAdditionalConfig
|
||||
|
||||
SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe"
|
||||
set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar"
|
||||
set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
|
||||
|
||||
set WRAPPER_URL="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar"
|
||||
|
||||
FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO (
|
||||
IF "%%A"=="wrapperUrl" SET WRAPPER_URL=%%B
|
||||
)
|
||||
|
||||
@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
|
||||
@REM This allows using the maven wrapper in projects that prohibit checking in binary data.
|
||||
if exist %WRAPPER_JAR% (
|
||||
if "%MVNW_VERBOSE%" == "true" (
|
||||
echo Found %WRAPPER_JAR%
|
||||
)
|
||||
) else (
|
||||
if not "%MVNW_REPOURL%" == "" (
|
||||
SET WRAPPER_URL="%MVNW_REPOURL%/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar"
|
||||
)
|
||||
if "%MVNW_VERBOSE%" == "true" (
|
||||
echo Couldn't find %WRAPPER_JAR%, downloading it ...
|
||||
echo Downloading from: %WRAPPER_URL%
|
||||
)
|
||||
|
||||
powershell -Command "&{"^
|
||||
"$webclient = new-object System.Net.WebClient;"^
|
||||
"if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^
|
||||
"$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^
|
||||
"}"^
|
||||
"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%WRAPPER_URL%', '%WRAPPER_JAR%')"^
|
||||
"}"
|
||||
if "%MVNW_VERBOSE%" == "true" (
|
||||
echo Finished downloading %WRAPPER_JAR%
|
||||
)
|
||||
)
|
||||
@REM End of extension
|
||||
|
||||
@REM If specified, validate the SHA-256 sum of the Maven wrapper jar file
|
||||
SET WRAPPER_SHA_256_SUM=""
|
||||
FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO (
|
||||
IF "%%A"=="wrapperSha256Sum" SET WRAPPER_SHA_256_SUM=%%B
|
||||
)
|
||||
IF NOT %WRAPPER_SHA_256_SUM%=="" (
|
||||
powershell -Command "&{"^
|
||||
"$hash = (Get-FileHash \"%WRAPPER_JAR%\" -Algorithm SHA256).Hash.ToLower();"^
|
||||
"If('%WRAPPER_SHA_256_SUM%' -ne $hash){"^
|
||||
" Write-Output 'Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised.';"^
|
||||
" Write-Output 'Investigate or delete %WRAPPER_JAR% to attempt a clean download.';"^
|
||||
" Write-Output 'If you updated your Maven version, you need to update the specified wrapperSha256Sum property.';"^
|
||||
" exit 1;"^
|
||||
"}"^
|
||||
"}"
|
||||
if ERRORLEVEL 1 goto error
|
||||
)
|
||||
|
||||
@REM Provide a "standardized" way to retrieve the CLI args that will
|
||||
@REM work with both Windows and non-Windows executions.
|
||||
set MAVEN_CMD_LINE_ARGS=%*
|
||||
|
||||
%MAVEN_JAVA_EXE% ^
|
||||
%JVM_CONFIG_MAVEN_PROPS% ^
|
||||
%MAVEN_OPTS% ^
|
||||
%MAVEN_DEBUG_OPTS% ^
|
||||
-classpath %WRAPPER_JAR% ^
|
||||
"-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" ^
|
||||
%WRAPPER_LAUNCHER% %MAVEN_CONFIG% %*
|
||||
if ERRORLEVEL 1 goto error
|
||||
goto end
|
||||
|
||||
:error
|
||||
set ERROR_CODE=1
|
||||
|
||||
:end
|
||||
@endlocal & set ERROR_CODE=%ERROR_CODE%
|
||||
|
||||
if not "%MAVEN_SKIP_RC%"=="" goto skipRcPost
|
||||
@REM check for post script, once with legacy .bat ending and once with .cmd ending
|
||||
if exist "%USERPROFILE%\mavenrc_post.bat" call "%USERPROFILE%\mavenrc_post.bat"
|
||||
if exist "%USERPROFILE%\mavenrc_post.cmd" call "%USERPROFILE%\mavenrc_post.cmd"
|
||||
:skipRcPost
|
||||
|
||||
@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on'
|
||||
if "%MAVEN_BATCH_PAUSE%"=="on" pause
|
||||
|
||||
if "%MAVEN_TERMINATE_CMD%"=="on" exit %ERROR_CODE%
|
||||
|
||||
cmd /C exit /B %ERROR_CODE%
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>de.hsma</groupId>
|
||||
<artifactId>spark-it</artifactId>
|
||||
<version>0.0.1-SNAPSHOT</version>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.target>11</maven.compiler.target>
|
||||
<maven.compiler.source>11</maven.compiler.source>
|
||||
<spark.version>3.5.1</spark.version>
|
||||
<cassandra.version>3.5.0</cassandra.version>
|
||||
<slf4j.version>1.7.36</slf4j.version>
|
||||
<jackson.version>2.15.2</jackson.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-core_2.13</artifactId>
|
||||
<version>${spark.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-sql_2.13</artifactId>
|
||||
<version>${spark.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-streaming_2.13</artifactId>
|
||||
<version>${spark.version}</version>
|
||||
<!-- <scope>provided</scope> -->
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-mllib_2.13</artifactId>
|
||||
<version>${spark.version}</version>
|
||||
<!-- <scope>provided</scope> -->
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.datastax.spark</groupId>
|
||||
<artifactId>spark-cassandra-connector_2.12</artifactId>
|
||||
<version>${cassandra.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-core</artifactId>
|
||||
<version>${jackson.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
<version>${jackson.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
<version>${slf4j.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
package de.hsma.informatik.bdea;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.spark.sql.Dataset;
|
||||
import org.apache.spark.sql.Encoders;
|
||||
import org.apache.spark.sql.SparkSession;
|
||||
|
||||
public class BasicDataset {
|
||||
|
||||
public static void main(String[] args) {
|
||||
SparkSession spark = SparkSession
|
||||
.builder()
|
||||
.appName("DSExample").master("local[4]")
|
||||
.getOrCreate();
|
||||
|
||||
List<Integer> data = Arrays.asList(2, 3, 4, 6, 7, 8);
|
||||
Dataset<Integer> ds = spark.createDataset(data, Encoders.INT());
|
||||
|
||||
ds.createOrReplaceTempView("numbers");
|
||||
spark.sql("select * from numbers where value > 3").show();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
package de.hsma.informatik.bdea;
|
||||
|
||||
import static org.apache.spark.sql.functions.col;
|
||||
|
||||
import org.apache.spark.sql.Dataset;
|
||||
import org.apache.spark.sql.Encoders;
|
||||
import org.apache.spark.sql.Row;
|
||||
import org.apache.spark.sql.SparkSession;
|
||||
|
||||
// https://github.com/apache/spark/blob/master/examples/src/main/java/org/apache/spark/examples/sql/JavaSparkSQLExample.java
|
||||
|
||||
public class BasicSQL {
|
||||
|
||||
public static void main(String[] args) {
|
||||
SparkSession spark = SparkSession.builder()
|
||||
.appName("Spark SQL examples").master("local[*]").getOrCreate();
|
||||
|
||||
Dataset<Row> data = spark.read().json("src/main/resources/data.json");
|
||||
|
||||
data.show();
|
||||
data.printSchema();
|
||||
|
||||
Dataset<Row> fnameDS = data.select("fname");
|
||||
fnameDS.show();
|
||||
|
||||
Dataset<Row> nameDS = data.select(col("fname"), col("lname"));
|
||||
nameDS.show();
|
||||
|
||||
Dataset<Row> genderDS = data.groupBy("gender").count();
|
||||
genderDS.show();
|
||||
|
||||
Dataset<Row> maleGenderDS = data.filter(col("gender").eqNullSafe("Male"));
|
||||
maleGenderDS.show();
|
||||
|
||||
Dataset<Person> peopleDS = data.as(Encoders.bean(Person.class));
|
||||
peopleDS.show();
|
||||
|
||||
System.out.println("Select...");
|
||||
data.createOrReplaceTempView("People");
|
||||
Dataset<Row> sqlDF = spark.sql("SELECT * FROM People WHERE lname like 'W%'");
|
||||
sqlDF.show();
|
||||
|
||||
spark.close();
|
||||
spark.stop();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,117 @@
|
|||
package de.hsma.informatik.bdea;
|
||||
|
||||
import static org.apache.spark.sql.functions.dayofyear;
|
||||
import static org.apache.spark.sql.functions.month;
|
||||
import static org.apache.spark.sql.functions.unix_timestamp;
|
||||
import static org.apache.spark.sql.functions.year;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
import org.apache.spark.sql.Dataset;
|
||||
import org.apache.spark.sql.Row;
|
||||
import org.apache.spark.sql.SparkSession;
|
||||
|
||||
// vgl.: https://databricks-prod-cloudfront.cloud.databricks.com/public/4027ec902e239c93eaaa8714f173bcfc/8599738367597028/4332292154849829/3601578643761083/latest.html
|
||||
// Daten von: https://data.sfgov.org/Public-Safety/Fire-Department-Calls-for-Service/nuek-vuh3
|
||||
// https://www.youtube.com/watch?v=K14plpZgy_c
|
||||
// Update 24.04.24: copy is here: https://github.com/NashTech-Labs/Sparkathon/blob/master/src/main/resources/assignment/Fire_Department_Calls_for_Service.csv
|
||||
public class FriscoFire {
|
||||
|
||||
public static void main(String[] args) {
|
||||
SparkSession spark = SparkSession.builder()
|
||||
.appName("Spark SQL examples").master("local[4]").getOrCreate();
|
||||
|
||||
Dataset<Row> data = spark.read().format("csv").option("header", true).option("inferSchema", true)
|
||||
.load("/home/marcus/Downloads/Fire_Department_Calls_for_Service.csv");
|
||||
|
||||
// Mal schauen, was wir da geladen haben
|
||||
data.printSchema();
|
||||
data.limit(5).show();
|
||||
|
||||
// Spalten umbenennen (Leerzeichen und Bindestrich für spätere SQL-Abfragen entfernen)
|
||||
for (String name : data.columns()) {
|
||||
data = data.withColumnRenamed(name, name.replaceAll(" ", ""));
|
||||
}
|
||||
data = data.withColumnRenamed("Neighborhooods-AnalysisBoundaries", "Neighborhoods");
|
||||
|
||||
// // Spaltennamen ausgeben
|
||||
Arrays.stream(data.columns()).forEach(System.out::println);
|
||||
|
||||
// // Anzahl der Zeilen
|
||||
System.out.println(data.count());
|
||||
//
|
||||
// // Arten von Notfällen, do not truncate output
|
||||
data.select("CallType").distinct().show(false);
|
||||
|
||||
// // Wie viele Anrufe welchen Typs gab es?
|
||||
data.select("CallType").groupBy("CallType").count().orderBy("count").show(false);
|
||||
|
||||
// Datentypen anpassen: String zu Timestamp konvertieren
|
||||
String fp1 = "MM/dd/yyyy";
|
||||
String fp2 = "MM/dd/yyyy hh:mm:ss a";
|
||||
|
||||
data = data.withColumn("CallDateTS", unix_timestamp(data.col("CallDate"), fp1).cast("timestamp")).drop("CallDate");
|
||||
data = data.withColumn("WatchDateTS", unix_timestamp(data.col("WatchDate"), fp1).cast("timestamp")).drop("WatchDate");
|
||||
|
||||
data = data.withColumn("ReceivedDtTmTS", unix_timestamp(data.col("ReceivedDtTm"), fp2).cast("timestamp")).drop("ReceivedDtTm");
|
||||
data = data.withColumn("EntryDtTmTS", unix_timestamp(data.col("EntryDtTm"), fp2).cast("timestamp")).drop("EntryDtTm");
|
||||
data = data.withColumn("DispatchDtTmTS", unix_timestamp(data.col("DispatchDtTm"), fp2).cast("timestamp")).drop("DispatchDtTm");
|
||||
data = data.withColumn("ResponseDtTmTS", unix_timestamp(data.col("ResponseDtTm"), fp2).cast("timestamp")).drop("ResponseDtTm");
|
||||
data = data.withColumn("OnSceneDtTmTS", unix_timestamp(data.col("OnSceneDtTm"), fp2).cast("timestamp")).drop("OnSceneDtTm");
|
||||
data = data.withColumn("TransportDtTmTS", unix_timestamp(data.col("TransportDtTm"), fp2).cast("timestamp")).drop("TransportDtTm");
|
||||
data = data.withColumn("HospitalDtTmTS", unix_timestamp(data.col("HospitalDtTm"), fp2).cast("timestamp")).drop("HospitalDtTm");
|
||||
data = data.withColumn("AvailableDtTmTS", unix_timestamp(data.col("AvailableDtTm"), fp2).cast("timestamp")).drop("AvailableDtTm");
|
||||
|
||||
// neues Schema anschauen
|
||||
data.printSchema();
|
||||
data.limit(5).show();
|
||||
|
||||
// welche Anrufgründe gibt es?
|
||||
data.select(data.col("CallType")).distinct().orderBy(year(data.col("CallDateTS"))).show(100, false);
|
||||
|
||||
// welche Jahre an Daten haben wir?
|
||||
data.select(year(data.col("CallDateTS"))).distinct().orderBy(data.col("CallDateTS")).show();
|
||||
|
||||
// zeige die Anrufe aus 2018
|
||||
data.filter(year(data.col("CallDateTS")).equalTo("2018")).show();
|
||||
|
||||
// wie viele Anrufe gab es im Juli 2017?
|
||||
long calls = data.filter(year(data.col("CallDateTS")).equalTo("2017"))
|
||||
.filter(month(data.col("CallDateTS")).equalTo("07")).count();
|
||||
System.out.println(calls);
|
||||
|
||||
// wie viele Anrufe in 7 Tagen in 2016?
|
||||
data.filter(year(data.col("CallDateTS")).equalTo("2016"))
|
||||
.filter(dayofyear(data.col("CallDateTS")).$greater("179"))
|
||||
.groupBy(dayofyear(data.col("CallDateTS"))).count().orderBy(dayofyear(data.col("CallDateTS"))).show(7);
|
||||
|
||||
data.repartition(4).createOrReplaceTempView("fireServiceVIEW");
|
||||
spark.sql("SELECT DISTINCT Neighborhoods FROM fireServiceVIEW LIMIT 15").show();
|
||||
|
||||
// Cachen kostet erst einmal deutlich mehr Zeit...
|
||||
// spark.catalog().cacheTable("fireServiceVIEW");
|
||||
// System.out.println("----> Count: " + spark.table("fireServiceVIEW").count());
|
||||
|
||||
// // mit SQL: woher kamen 2015 die meisten Anrufe?
|
||||
spark.sql("SELECT Neighborhoods, count(Neighborhoods) AS Neighborhood_Count FROM fireServiceVIEW "
|
||||
+ "WHERE year(CallDateTS) == 2015 GROUP BY Neighborhoods ORDER BY Neighborhood_Count DESC LIMIT 15").show();
|
||||
|
||||
// Joinen... zweite Tabelle: https://data.sfgov.org/Public-Safety/Fire-Incidents/wr8u-xric
|
||||
Dataset<Row> incidents = spark.read().format("csv").option("header", true).option("inferSchema", true)
|
||||
.load("/home/marcus/Downloads/Fire_Incidents.csv").withColumnRenamed("Incident Number", "IncidentNumber").cache();
|
||||
|
||||
System.out.println(incidents.count());
|
||||
incidents.show(3);
|
||||
|
||||
// Hier passiert der Join... kann einen Moment dauern
|
||||
// s. auch: http://www.techburps.com/misc/apache-spark-dataset-joins-in-java/129
|
||||
System.out.println("Joining...");
|
||||
Dataset<Row> joined = data.join(incidents, data.col("IncidentNumber").equalTo(incidents.col("IncidentNumber")));
|
||||
// Dataset<Row> joined = data.join(incidents, data.col("IncidentNumber").equalTo(incidents.col("IncidentNumber")), "fullouter");
|
||||
joined.show(100);
|
||||
|
||||
|
||||
System.out.println("Fin.");
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
package de.hsma.informatik.bdea;
|
||||
|
||||
import org.apache.spark.SparkConf;
|
||||
import org.apache.spark.api.java.JavaRDD;
|
||||
import org.apache.spark.api.java.JavaSparkContext;
|
||||
|
||||
public class LineCount {
|
||||
|
||||
public static void main(String[] args) {
|
||||
String pfad = "src/main/resources/Faust.txt";
|
||||
|
||||
SparkConf conf = new SparkConf()
|
||||
.setAppName("LineCount").setMaster("local[*]");
|
||||
JavaSparkContext sc = new JavaSparkContext(conf);
|
||||
|
||||
JavaRDD<String> text = sc.textFile(pfad);
|
||||
long cnt = text.filter(line -> line.contains("Liebe")).count();
|
||||
|
||||
System.out.println(cnt + " Zeilen mit 'Liebe' in " + pfad);
|
||||
sc.close();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
package de.hsma.informatik.bdea;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
@JsonIgnoreProperties
|
||||
public class Person implements Serializable {
|
||||
|
||||
@JsonProperty(value = "id")
|
||||
private Long id;
|
||||
|
||||
@JsonProperty(value = "fname")
|
||||
private String fname;
|
||||
|
||||
@JsonProperty(value = "lname")
|
||||
private String lname;
|
||||
|
||||
@JsonProperty(value = "email")
|
||||
private String email;
|
||||
|
||||
@JsonProperty(value = "gender")
|
||||
private String gender;
|
||||
|
||||
public Long getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(Long id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public String getFname() {
|
||||
return fname;
|
||||
}
|
||||
|
||||
public void setFname(String fname) {
|
||||
this.fname = fname;
|
||||
}
|
||||
|
||||
public String getLname() {
|
||||
return lname;
|
||||
}
|
||||
|
||||
public void setLname(String lname) {
|
||||
this.lname = lname;
|
||||
}
|
||||
|
||||
public String getEmail() {
|
||||
return email;
|
||||
}
|
||||
|
||||
public void setEmail(String email) {
|
||||
this.email = email;
|
||||
}
|
||||
|
||||
public String getGender() {
|
||||
return gender;
|
||||
}
|
||||
|
||||
public void setGender(String gender) {
|
||||
this.gender = gender;
|
||||
}
|
||||
|
||||
//getters and setters
|
||||
}
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
package de.hsma.informatik.bdea;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.spark.SparkConf;
|
||||
import org.apache.spark.api.java.JavaSparkContext;
|
||||
|
||||
public class PiExample {
|
||||
private final static int NUM_SAMPLES = 10000000;
|
||||
|
||||
public static void main(String[] args) {
|
||||
SparkConf conf = new SparkConf()
|
||||
.setAppName("Getting-Started").setMaster("local[*]");
|
||||
JavaSparkContext sc = new JavaSparkContext(conf);
|
||||
|
||||
List<Integer> l = new ArrayList<>(NUM_SAMPLES);
|
||||
for (int i = 0; i < NUM_SAMPLES; i++) {
|
||||
l.add(i);
|
||||
}
|
||||
|
||||
long count = sc.parallelize(l).filter(i -> {
|
||||
double x = Math.random();
|
||||
double y = Math.random();
|
||||
return x*x + y*y <= 1;
|
||||
}).count();
|
||||
|
||||
System.out.println("Pi is roughly " + 4.0 * count / NUM_SAMPLES);
|
||||
|
||||
sc.close();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
package de.hsma.informatik.bdea;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
import org.apache.spark.SparkConf;
|
||||
import org.apache.spark.api.java.JavaRDD;
|
||||
import org.apache.spark.api.java.JavaSparkContext;
|
||||
|
||||
public class SimpleAdder {
|
||||
|
||||
public static void main(String[] args) {
|
||||
SparkConf conf = new SparkConf().setAppName("SimpleAdder").setMaster("local[2]");
|
||||
JavaSparkContext sc = new JavaSparkContext(conf);
|
||||
|
||||
Integer[] ints = {1,2,4,5,6,7,8,9};
|
||||
JavaRDD<Integer> dist = sc.parallelize(Arrays.asList(ints));
|
||||
|
||||
dist.collect().forEach(System.out::println);
|
||||
|
||||
|
||||
System.out.println(dist.reduce((x,y) -> x+y));
|
||||
|
||||
|
||||
Integer[] others = {2,3,1,0, 0, 3, 4};
|
||||
JavaRDD<Integer> dist2 = sc.parallelize(Arrays.asList(others));
|
||||
// JavaRDD<Integer> res = dist.intersection(dist2);
|
||||
// JavaPairRDD<Integer, Integer> res = dist.join(dist2);
|
||||
// res.collect().forEach(System.out::println);
|
||||
|
||||
dist.intersection(dist2).collect().forEach(System.out::println);
|
||||
|
||||
dist2 = dist2.sortBy(x->x, true, 1);
|
||||
|
||||
dist2.sample(true, 0.5).collect().forEach(System.out::println);
|
||||
// dist2.take(2).forEach(System.out::println);
|
||||
dist2.collect().forEach(System.out::println);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
package de.hsma.informatik.bdea;
|
||||
|
||||
import org.apache.spark.api.java.function.FlatMapFunction;
|
||||
import org.apache.spark.sql.Dataset;
|
||||
import org.apache.spark.sql.Encoders;
|
||||
import org.apache.spark.sql.Row;
|
||||
import org.apache.spark.sql.SparkSession;
|
||||
import org.apache.spark.sql.streaming.StreamingQuery;
|
||||
import org.apache.spark.sql.streaming.StreamingQueryException;
|
||||
import org.apache.spark.sql.streaming.Trigger;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
// from https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html
|
||||
public class SparkStructuredStreaming {
|
||||
|
||||
public static void main(String[] args) throws InterruptedException, TimeoutException, StreamingQueryException {
|
||||
SparkSession spark = SparkSession
|
||||
.builder()
|
||||
.appName("JavaStructuredNetworkWordCount")
|
||||
.master("local[*]")
|
||||
.getOrCreate();
|
||||
|
||||
// Folgendes benötigt einen lokalen Server auf Port 9999
|
||||
// z.B. nc -lk 9999
|
||||
|
||||
// Create DataFrame representing the stream of input lines from connection to localhost:9999
|
||||
Dataset<Row> lines = spark
|
||||
.readStream()
|
||||
.format("socket")
|
||||
.option("host", "localhost")
|
||||
.option("port", 9999)
|
||||
.load();
|
||||
|
||||
// Split the lines into words
|
||||
Dataset<String> words = lines
|
||||
.as(Encoders.STRING())
|
||||
.flatMap((FlatMapFunction<String, String>) x -> Arrays.asList(x.split(" ")).iterator(), Encoders.STRING());
|
||||
|
||||
// Generate running word count
|
||||
Dataset<Row> wordCounts = words.groupBy("value").count();
|
||||
|
||||
// Start running the query that prints the running counts to the console
|
||||
StreamingQuery query = wordCounts.writeStream()
|
||||
.outputMode("complete")
|
||||
.format("console")
|
||||
// https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html#triggers
|
||||
.trigger(Trigger.ProcessingTime("5 seconds")) // interval
|
||||
.start();
|
||||
|
||||
query.awaitTermination();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
package de.hsma.informatik.bdea;
|
||||
|
||||
import org.apache.spark.api.java.function.FlatMapFunction;
|
||||
import org.apache.spark.sql.*;
|
||||
import org.apache.spark.sql.streaming.StreamingQuery;
|
||||
import org.apache.spark.sql.streaming.StreamingQueryException;
|
||||
|
||||
import scala.Tuple2;
|
||||
|
||||
import java.sql.Timestamp;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
// from https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html
|
||||
public class SparkStructuredStreamingWindowed {
|
||||
|
||||
public static void main(String[] args) throws InterruptedException, TimeoutException, StreamingQueryException {
|
||||
SparkSession spark = SparkSession
|
||||
.builder()
|
||||
.appName("JavaStructuredNetworkWordCount")
|
||||
.master("local[*]")
|
||||
.getOrCreate();
|
||||
|
||||
int windowSize = 10;
|
||||
int slideSize = 5;
|
||||
|
||||
// 'windowSize' Sekunden Windows
|
||||
String windowDuration = windowSize + " seconds";
|
||||
// werden alle 'slideSize' Sekunden aufgefrischt
|
||||
String slideDuration = slideSize + " seconds";
|
||||
|
||||
// Folgendes benötigt einen lokalen Server auf Port 9999
|
||||
// z.B. nc -lk 9999
|
||||
|
||||
// Create DataFrame representing the stream of input lines from connection to localhost:9999
|
||||
Dataset<Row> lines = spark
|
||||
.readStream()
|
||||
.format("socket")
|
||||
.option("host", "localhost")
|
||||
.option("port", 9999)
|
||||
.option("includeTimestamp", true) // Timestamps!
|
||||
.load();
|
||||
|
||||
// Split the lines into words, retaining timestamps
|
||||
Dataset<Row> words = lines
|
||||
.as(Encoders.tuple(Encoders.STRING(), Encoders.TIMESTAMP()))
|
||||
.flatMap((FlatMapFunction<Tuple2<String, Timestamp>, Tuple2<String, Timestamp>>) t -> {
|
||||
List<Tuple2<String, Timestamp>> result = new ArrayList<>();
|
||||
for (String word : t._1.split(" ")) {
|
||||
result.add(new Tuple2<>(word, t._2));
|
||||
}
|
||||
return result.iterator();
|
||||
},
|
||||
Encoders.tuple(Encoders.STRING(), Encoders.TIMESTAMP())
|
||||
).toDF("word", "timestamp");
|
||||
|
||||
// Group the data by window and word and compute the count of each group
|
||||
Dataset<Row> windowedCounts = words.groupBy(
|
||||
functions.window(words.col("timestamp"), windowDuration, slideDuration),
|
||||
words.col("word")
|
||||
).count().orderBy("window");
|
||||
|
||||
// Start running the query that prints the windowed word counts to the console
|
||||
StreamingQuery query = windowedCounts.writeStream()
|
||||
.outputMode("complete")
|
||||
.format("console")
|
||||
.option("truncate", "false")
|
||||
.start();
|
||||
|
||||
query.awaitTermination();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
package de.hsma.informatik.bdea;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.spark.SparkConf;
|
||||
import org.apache.spark.api.java.JavaPairRDD;
|
||||
import org.apache.spark.api.java.JavaRDD;
|
||||
import org.apache.spark.api.java.JavaSparkContext;
|
||||
|
||||
import scala.Tuple2;
|
||||
|
||||
public class WordCount {
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
||||
SparkConf conf = new SparkConf().setAppName("wc").setMaster("local[4]");
|
||||
JavaSparkContext sc = new JavaSparkContext(conf);
|
||||
|
||||
JavaRDD<String> tokens = sc.textFile("src/main/resources/Faust.txt").flatMap(
|
||||
s -> Arrays.asList(s.split("\\W+")).iterator());
|
||||
|
||||
JavaPairRDD<String, Integer> counts = tokens.mapToPair(
|
||||
token -> new Tuple2<>(token, 1)).reduceByKey((x,y) -> x+y);
|
||||
|
||||
List<Tuple2<String, Integer>> results = counts.collect();
|
||||
results.forEach(System.out::println);
|
||||
|
||||
sc.close();
|
||||
}
|
||||
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue