fork download

copy

 
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.broadcast.Broadcast;
import scala.Tuple2;
 
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.util.*;
 
/**
 * Created by 11130 on 14/02/16.
 */
public class MostPopularSuperHero {
 
    public static void main(String args[]) {
        SparkConf conf = new SparkConf().setAppName("MostPopularSuperHero").setMaster("local[*]");
        JavaSparkContext sc = new JavaSparkContext(conf);
 
        class HrDict {
            Map<Integer, String> getHeroDict() {
                Map<Integer, String> heroDict = new HashMap<>();
                BufferedReader br = null;
 
                try {
 
                    String sCurrentLine;
 
                    br = new BufferedReader(new FileReader("/Users/11130/udemy/SparkCourse/Marvel-Names.txt"));
 
                    while ((sCurrentLine = br.readLine()) != null) {
                        String str = sCurrentLine;
                        String[] fields = str.split(" ", 2);
                        heroDict.put(Integer.parseInt(fields[0]), fields[1]);
                    }
 
                } catch (IOException e) {
                    e.printStackTrace();
                }
 
                return heroDict;
            }
        }
 
        class DummyComparator implements Comparator<Tuple2<Integer, String> > {
            @Override
            public int compare(Tuple2<Integer, String> o1, Tuple2<Integer, String> o2) {
                return Integer.compare(o1._1(), o2._1());
            }
        }
 
        Broadcast<Map<Integer, String> > heroDict = sc.broadcast(new HrDict().getHeroDict());
        JavaRDD<String> lines = sc.textFile("/Users/11130/udemy/SparkCourse/Marvel-Graph.txt");
 
        JavaPairRDD<Integer, Integer> countOfOccurences = lines.mapToPair(
                s -> {
                    String[] heroes = s.split(" ");
                    return new Tuple2<>(Integer.parseInt(heroes[0]), heroes.length - 1);
                }
        ).reduceByKey(
                (x, y) -> x + y
        );
 
        JavaPairRDD<Integer, String> flippedCountOfOccurences = countOfOccurences.mapToPair(
                s -> new Tuple2<>(s._2(), heroDict.getValue().get(s._1()))
        );
 
 
 
        Tuple2<Integer, String> result = flippedCountOfOccurences.max(new DummyComparator());
 
        System.out.println("The most populat superhero is " + result._2() + " with " + result._1() + " number of occurences");
 
    }
}
 

Main.java:17: error: class MostPopularSuperHero is public, should be declared in a file named MostPopularSuperHero.java
public class MostPopularSuperHero {
       ^
Main.java:2: error: package org.apache.spark does not exist
import org.apache.spark.SparkConf;
                       ^
Main.java:3: error: package org.apache.spark.api.java does not exist
import org.apache.spark.api.java.JavaPairRDD;
                                ^
Main.java:4: error: package org.apache.spark.api.java does not exist
import org.apache.spark.api.java.JavaRDD;
                                ^
Main.java:5: error: package org.apache.spark.api.java does not exist
import org.apache.spark.api.java.JavaSparkContext;
                                ^
Main.java:6: error: package org.apache.spark.broadcast does not exist
import org.apache.spark.broadcast.Broadcast;
                                 ^
Main.java:7: error: package scala does not exist
import scala.Tuple2;
            ^
Main.java:19: error: cannot find symbol
    static class DummyComparator implements Comparator<Tuple2<Integer, String>> {
                                                       ^
  symbol:   class Tuple2
  location: class MostPopularSuperHero
Main.java:20: error: cannot find symbol
        public int compare(Tuple2<Integer, String> x, Tuple2<Integer, String> y) {
                           ^
  symbol:   class Tuple2
  location: class DummyComparator
Main.java:20: error: cannot find symbol
        public int compare(Tuple2<Integer, String> x, Tuple2<Integer, String> y) {
                                                      ^
  symbol:   class Tuple2
  location: class DummyComparator
Main.java:49: error: cannot find symbol
        SparkConf conf = new SparkConf().setAppName("MostPopularSuperHero").setMaster("local[*]");
        ^
  symbol:   class SparkConf
  location: class MostPopularSuperHero
Main.java:49: error: cannot find symbol
        SparkConf conf = new SparkConf().setAppName("MostPopularSuperHero").setMaster("local[*]");
                             ^
  symbol:   class SparkConf
  location: class MostPopularSuperHero
Main.java:50: error: cannot find symbol
        JavaSparkContext sc = new JavaSparkContext(conf);
        ^
  symbol:   class JavaSparkContext
  location: class MostPopularSuperHero
Main.java:50: error: cannot find symbol
        JavaSparkContext sc = new JavaSparkContext(conf);
                                  ^
  symbol:   class JavaSparkContext
  location: class MostPopularSuperHero
Main.java:51: error: cannot find symbol
        Broadcast<Map<Integer, String> > heroDict = sc.broadcast(getHeroDict());
        ^
  symbol:   class Broadcast
  location: class MostPopularSuperHero
Main.java:52: error: cannot find symbol
        JavaRDD<String> lines = sc.textFile("/Users/11130/udemy/SparkCourse/Marvel-Graph.txt");
        ^
  symbol:   class JavaRDD
  location: class MostPopularSuperHero
Main.java:54: error: cannot find symbol
        JavaPairRDD<Integer, Integer> countOfOccurences = lines.mapToPair(
        ^
  symbol:   class JavaPairRDD
  location: class MostPopularSuperHero
Main.java:57: error: cannot find symbol
                    return new Tuple2<>(Integer.parseInt(heroes[0]), heroes.length - 1);
                               ^
  symbol:   class Tuple2
  location: class MostPopularSuperHero
Main.java:63: error: cannot find symbol
        JavaPairRDD<Integer, String> flippedCountOfOccurences = countOfOccurences.mapToPair(
        ^
  symbol:   class JavaPairRDD
  location: class MostPopularSuperHero
Main.java:64: error: cannot find symbol
                s -> new Tuple2<>(s._2(), heroDict.getValue().get(s._1()))
                         ^
  symbol:   class Tuple2
  location: class MostPopularSuperHero
Main.java:67: error: cannot find symbol
        Tuple2<Integer, String> result = flippedCountOfOccurences.max(new DummyComparator());
        ^
  symbol:   class Tuple2
  location: class MostPopularSuperHero
21 errors

Standard output is empty