File tree

• .gitattributes
• .gitignore
• .rat-excludes
• LICENSE
• README.md
• assembly
  • pom.xml
• bagel
  • pom.xml
  • src/test/resources
    • log4j.properties
• bin
  • beeline.cmd
  • compute-classpath.cmd
  • compute-classpath.sh
  • load-spark-env.sh
  • pyspark2.cmd
  • pyspark
  • run-example
  • spark-class
  • spark-shell2.cmd
  • spark-shell
  • spark-sql
  • spark-submit2.cmd
  • spark-submit
  • utils.sh
  • windows-utils.cmd
• build
  • mvn
  • sbt-launch-lib.bash
  • sbt
• conf
  • metrics.properties.template
  • spark-env.sh.template
• core
  • pom.xml
  • src
    • main
      • java/org/apache/spark
        • JavaSparkListener.java
        • JobExecutionStatus.java
        • SparkFirehoseListener.java
        • SparkJobInfo.java
        • SparkStageInfo.java
        • TaskContext.java
        • api/java/function
          • PairFunction.java
          • package.scala
        • util/collection
          • Sorter.java
          • TimSort.java
      • resources/org/apache/spark/ui/static
        • additional-metrics.js
        • table.js
        • webui.css
      • scala/org/apache/spark
        • Accumulators.scala
        • Aggregator.scala
        • CacheManager.scala
        • Dependency.scala
        • ExecutorAllocationClient.scala
        • ExecutorAllocationManager.scala
        • FutureAction.scala
        • HttpFileServer.scala
        • HttpServer.scala
        • Logging.scala
        • MapOutputTracker.scala
        • Partition.scala
        • Partitioner.scala
        • SSLOptions.scala
        • SecurityManager.scala
        • SerializableWritable.scala
        • SparkConf.scala
        • SparkContext.scala
        • SparkEnv.scala
        • SparkHadoopWriter.scala
        • SparkSaslClient.scala
        • SparkSaslServer.scala
        • SparkStatusTracker.scala
        • StatusAPIImpl.scala
        • TaskContext.scala
        • TaskContextImpl.scala
        • TaskEndReason.scala
        • TaskNotSerializableException.scala
        • TestUtils.scala
        • api
          • java
            • JavaDoubleRDD.scala
            • JavaPairRDD.scala
            • JavaRDDLike.scala
            • JavaSparkContext.scala
            • JavaSparkStatusTracker.scala
            • JavaUtils.scala
          • python
            • PythonHadoopUtil.scala
            • PythonRDD.scala
            • PythonUtils.scala
            • SerDeUtil.scala
            • WriteInputFormatTestDataGenerator.scala
        • broadcast
          • Broadcast.scala
          • HttpBroadcast.scala
          • TorrentBroadcast.scala
        • deploy
          • ApplicationDescription.scala
          • Client.scala
          • ClientArguments.scala
          • DeployMessage.scala
          • DriverDescription.scala
          • PythonRunner.scala
          • SparkHadoopUtil.scala
          • SparkSubmit.scala
          • SparkSubmitArguments.scala
          • SparkSubmitDriverBootstrapper.scala
          • client
            • AppClient.scala
          • history
            • ApplicationHistoryProvider.scala
            • FsHistoryProvider.scala
            • HistoryPage.scala
            • HistoryServer.scala
            • HistoryServerArguments.scala
          • master
            • ApplicationInfo.scala
            • DriverInfo.scala
            • ExecutorDesc.scala
            • ExecutorInfo.scala
            • FileSystemPersistenceEngine.scala
            • LeaderElectionAgent.scala
            • Master.scala
            • PersistenceEngine.scala
            • RecoveryModeFactory.scala
            • WorkerInfo.scala
            • ZooKeeperLeaderElectionAgent.scala
            • ZooKeeperPersistenceEngine.scala
            • ui
              • ApplicationPage.scala
              • MasterWebUI.scala
          • worker
            • CommandUtils.scala
            • DriverRunner.scala
            • ExecutorRunner.scala
            • StandaloneWorkerShuffleService.scala
            • Worker.scala
            • ui
              • WorkerWebUI.scala
        • executor
          • CoarseGrainedExecutorBackend.scala
          • Executor.scala
          • ExecutorActor.scala
          • ExecutorExitCode.scala
          • ExecutorUncaughtExceptionHandler.scala
          • MesosExecutorBackend.scala
          • TaskMetrics.scala
        • input
          • FixedLengthBinaryInputFormat.scala
          • FixedLengthBinaryRecordReader.scala
          • PortableDataStream.scala
          • WholeTextFileInputFormat.scala
          • WholeTextFileRecordReader.scala
        • io
          • CompressionCodec.scala
        • mapreduce
          • SparkHadoopMapReduceUtil.scala
        • mapred
          • SparkHadoopMapRedUtil.scala
        • metrics
          • MetricsSystem.scala
          • sink
            • GraphiteSink.scala
        • network
          • BlockDataManager.scala
          • BlockFetchingListener.scala
          • BlockTransferService.scala
          • ManagedBuffer.scala
          • netty
            • NettyBlockRpcServer.scala
            • NettyBlockTransferService.scala
            • NettyConfig.scala
            • PathResolver.scala
            • SparkTransportConf.scala
            • client
              • BlockClientListener.scala
              • BlockFetchingClient.scala
              • BlockFetchingClientFactory.scala
              • BlockFetchingClientHandler.scala
              • LazyInitIterator.scala
              • ReferenceCountedBuffer.scala
            • server
              • BlockHeader.scala
              • BlockHeaderEncoder.scala
              • BlockServer.scala
              • BlockServerChannelInitializer.scala
              • BlockServerHandler.scala
          • nio
            • Connection.scala
            • ConnectionManager.scala
            • Message.scala
            • NioBlockTransferService.scala
        • package.scala
        • partial
          • CountEvaluator.scala
          • GroupedCountEvaluator.scala
          • MeanEvaluator.scala
          • StudentTCacher.scala
          • SumEvaluator.scala
        • rdd
          • AsyncRDDActions.scala
          • BinaryFileRDD.scala
          • BlockRDD.scala
          • CartesianRDD.scala
          • CheckpointRDD.scala
          • CoGroupedRDD.scala
          • CoalescedRDD.scala
          • DoubleRDDFunctions.scala
          • FilteredRDD.scala
          • FlatMappedRDD.scala
          • FlatMappedValuesRDD.scala
          • GlommedRDD.scala
          • HadoopRDD.scala
          • JdbcRDD.scala
          • MappedRDD.scala
          • MappedValuesRDD.scala
          • NewHadoopRDD.scala
          • OrderedRDDFunctions.scala
          • PairRDDFunctions.scala
          • ParallelCollectionRDD.scala
          • PartitionerAwareUnionRDD.scala
          • PipedRDD.scala
          • RDD.scala
          • SampledRDD.scala
          • SequenceFileRDDFunctions.scala
          • UnionRDD.scala
          • ZippedPartitionsRDD.scala
          • ZippedWithIndexRDD.scala
        • scheduler
          • DAGScheduler.scala
          • EventLoggingListener.scala
          • JobLogger.scala
          • LiveListenerBus.scala
          • MapStatus.scala
          • ReplayListenerBus.scala
          • SparkListener.scala
          • SparkListenerBus.scala
          • Stage.scala
          • Task.scala
          • TaskDescription.scala
          • TaskResult.scala
          • TaskResultGetter.scala
          • TaskScheduler.scala
          • TaskSchedulerImpl.scala
          • TaskSetManager.scala
          • cluster
            • CoarseGrainedClusterMessage.scala
            • CoarseGrainedSchedulerBackend.scala
            • ExecutorData.scala
            • ExecutorInfo.scala
            • SimrSchedulerBackend.scala
            • SparkDeploySchedulerBackend.scala
            • YarnSchedulerBackend.scala
            • mesos
              • CoarseMesosSchedulerBackend.scala
              • MesosSchedulerBackend.scala
              • MesosTaskLaunchData.scala
          • local
            • LocalBackend.scala
        • serializer
          • JavaSerializer.scala
          • KryoSerializer.scala
          • SerializationDebugger.scala
          • Serializer.scala
        • shuffle
          • FetchFailedException.scala
          • FileShuffleBlockManager.scala
          • IndexShuffleBlockManager.scala
          • ShuffleBlockManager.scala
          • ShuffleManager.scala
          • ShuffleMemoryManager.scala
          • ShuffleReader.scala
          • hash
            • BlockStoreShuffleFetcher.scala
            • HashShuffleReader.scala
            • HashShuffleWriter.scala
          • sort
            • SortShuffleManager.scala
            • SortShuffleWriter.scala
        • storage
          • BlockDataProvider.scala
          • BlockId.scala
          • BlockManager.scala
          • BlockManagerId.scala
          • BlockManagerMaster.scala
          • BlockManagerMasterActor.scala
          • BlockManagerMessages.scala
          • BlockNotFoundException.scala
          • BlockObjectWriter.scala
          • DiskBlockManager.scala
          • DiskStore.scala
          • MemoryStore.scala
          • ShuffleBlockFetcherIterator.scala
          • StorageLevel.scala
          • StorageStatusListener.scala
          • TachyonBlockManager.scala
          • TachyonStore.scala
        • ui
          • ConsoleProgressBar.scala
          • JettyUtils.scala
          • SparkUI.scala
          • ToolTips.scala
          • UIUtils.scala
          • UIWorkloadGenerator.scala
          • WebUI.scala
          • env
            • EnvironmentTab.scala
          • exec
            • ExecutorThreadDumpPage.scala
            • ExecutorsPage.scala
            • ExecutorsTab.scala
          • jobs
            • AllJobsPage.scala
            • AllStagesPage.scala
            • ExecutorTable.scala
            • JobPage.scala
            • JobProgressListener.scala
            • JobProgressPage.scala
            • JobProgressTab.scala
            • JobsTab.scala
            • PoolPage.scala
            • PoolTable.scala
            • StagePage.scala
            • StageTable.scala
            • StagesTab.scala
            • TaskDetailsClassNames.scala
            • UIData.scala
          • storage
            • RDDPage.scala
            • StoragePage.scala
            • StorageTab.scala
        • util
          • AkkaUtils.scala
          • AsynchronousListenerBus.scala
          • CompletionIterator.scala
          • EventLoop.scala
          • FileLogger.scala
          • JsonProtocol.scala
          • ListenerBus.scala
          • MetadataCleaner.scala
          • SerializableBuffer.scala
          • SparkExitCode.scala
          • SparkUncaughtExceptionHandler.scala
          • ThreadStackTrace.scala
          • Utils.scala
          • Vector.scala
          • collection
            • CompactBuffer.scala
            • ExternalAppendOnlyMap.scala
            • ExternalSorter.scala
            • SortDataFormat.scala
            • Sorter.scala
            • Spillable.scala
          • random
            • RandomSampler.scala
            • StratifiedSamplingUtils.scala
            • XORShiftRandom.scala
    • test
      • java
        • org/apache/spark
          • JavaAPISuite.java
          • JavaJdbcRDDSuite.java
        • test/org/apache/spark
          • JavaTaskCompletionListenerImpl.java
          • JavaTaskContextCompileCheck.java
      • resources
        • keystore
        • log4j.properties
        • truststore
        • untrusted-keystore
      • scala/org/apache
        • sparktest
          • ImplicitSuite.scala
        • spark
          • AccumulatorSuite.scala
          • CacheManagerSuite.scala
          • CheckpointSuite.scala
          • ContextCleanerSuite.scala
          • DistributedSuite.scala
          • DriverSuite.scala
          • ExecutorAllocationManagerSuite.scala
          • ExternalShuffleServiceSuite.scala
          • FailureSuite.scala
          • FileServerSuite.scala
          • FileSuite.scala
          • FutureActionSuite.scala
          • HashShuffleSuite.scala
          • ImplicitOrderingSuite.scala
          • JobCancellationSuite.scala
          • MapOutputTrackerSuite.scala
          • PartitioningSuite.scala
          • SSLOptionsSuite.scala
          • SSLSampleConfigs.scala
          • SecurityManagerSuite.scala
          • SharedSparkContext.scala
          • ShuffleNettySuite.scala
          • ShuffleSuite.scala
          • SortShuffleSuite.scala
          • SparkConfSuite.scala
          • SparkContextSchedulerCreationSuite.scala
          • SparkContextSuite.scala
          • StatusTrackerSuite.scala
          • api/python
            • PythonBroadcastSuite.scala
            • PythonRDDSuite.scala
            • SerDeUtilSuite.scala
          • broadcast
            • BroadcastSuite.scala
          • deploy
            • ClientSuite.scala
            • CommandUtilsSuite.scala
            • JsonProtocolSuite.scala
            • SparkSubmitSuite.scala
            • SparkSubmitUtilsSuite.scala
            • history
              • FsHistoryProviderSuite.scala
            • master
              • MasterSuite.scala
            • worker
              • ExecutorRunnerTest.scala
              • WorkerArgumentsTest.scala
              • WorkerSuite.scala
          • input
            • WholeTextFileRecordReaderSuite.scala
          • io
            • CompressionCodecSuite.scala
          • metrics
            • InputOutputMetricsSuite.scala
            • MetricsSystemSuite.scala
          • network
            • netty
              • NettyBlockTransferSecuritySuite.scala
              • ServerClientIntegrationSuite.scala
              • client
                • BlockFetchingClientHandlerSuite.scala
              • server
                • BlockHeaderEncoderSuite.scala
                • BlockServerHandlerSuite.scala
            • nio
              • ConnectionManagerSuite.scala
          • rdd
            • AsyncRDDActionsSuite.scala
            • DoubleRDDSuite.scala
            • JdbcRDDSuite.scala
            • PairRDDFunctionsSuite.scala
            • ParallelCollectionSplitSuite.scala
            • PipedRDDSuite.scala
            • RDDSuite.scala
            • SortingSuite.scala
          • scheduler
            • DAGSchedulerSuite.scala
            • EventLoggingListenerSuite.scala
            • MapStatusSuite.scala
            • NotSerializableFakeTask.scala
            • PoolSuite.scala
            • ReplayListenerSuite.scala
            • SparkListenerSuite.scala
            • SparkListenerWithClusterSuite.scala
            • TaskContextSuite.scala
            • TaskResultGetterSuite.scala
            • TaskSchedulerImplSuite.scala
            • TaskSetManagerSuite.scala
            • mesos
              • MesosSchedulerBackendSuite.scala
              • MesosTaskLaunchDataSuite.scala
          • serializer
            • KryoSerializerDistributedSuite.scala
            • KryoSerializerSuite.scala
            • SerializationDebuggerSuite.scala
            • TestSerializer.scala
          • shuffle
            • ShuffleMemoryManagerSuite.scala
            • hash
              • HashShuffleManagerSuite.scala
          • storage
            • BlockManagerReplicationSuite.scala
            • BlockManagerSuite.scala
            • LocalDirsSuite.scala
            • ShuffleBlockFetcherIteratorSuite.scala
          • ui
            • UISeleniumSuite.scala
            • jobs
              • JobProgressListenerSuite.scala
          • util
            • AkkaUtilsSuite.scala
            • CompletionIteratorSuite.scala
            • EventLoopSuite.scala
            • FileAppenderSuite.scala
            • FileLoggerSuite.scala
            • JsonProtocolSuite.scala
            • ResetSystemProperties.scala
            • SizeEstimatorSuite.scala
            • UtilsSuite.scala
            • collection
              • ExternalAppendOnlyMapSuite.scala
              • ExternalSorterSuite.scala
              • SorterSuite.scala
            • random
              • RandomSamplerSuite.scala
• dev
  • audit-release
    • audit_release.py
    • blank_sbt_build
      • build.sbt
    • sbt_app_hive
      • build.sbt
      • src/main/resources
        • hive-site.xml
  • change-version-to-2.10.sh
  • change-version-to-2.11.sh
  • check-license
  • create-release
    • create-release.sh
    • generate-contributors.py
    • known_translations
    • releaseutils.py
    • translate-contributors.py
  • merge_spark_pr.py
  • mima
  • run-tests-jenkins
  • run-tests
  • scalastyle
• docs
  • README.md
  • _config.yml
  • _layouts
    • global.html
  • _plugins
    • copy_api_dirs.rb
  • building-spark.md
  • configuration.md
  • ec2-scripts.md
  • graphx-programming-guide.md
  • hadoop-third-party-distributions.md
  • img
    • PIClusteringFiveCirclesInputsAndOutputs.png
    • data_parallel_vs_graph_parallel.png
    • graph_analytics_pipeline.png
    • ml-Pipeline.png
    • ml-PipelineModel.png
    • ml-Pipelines.pptx
    • tables_and_graphs.png
  • index.md
  • job-scheduling.md
  • ml-guide.md
  • mllib-clustering.md
  • mllib-collaborative-filtering.md
  • mllib-decision-tree.md
  • mllib-ensembles.md
  • mllib-feature-extraction.md
  • mllib-guide.md
  • mllib-linear-methods.md
  • mllib-naive-bayes.md
  • mllib-optimization.md
  • mllib-statistics.md
  • monitoring.md
  • programming-guide.md
  • quick-start.md
  • running-on-mesos.md
  • running-on-yarn.md
  • security.md
  • spark-standalone.md
  • sql-programming-guide.md
  • streaming-custom-receivers.md
  • streaming-flume-integration.md
  • streaming-kafka-integration.md
  • streaming-programming-guide.md