understanding_partitions.txt

understanding_partitions.txt

$ ./bin/pyspark
Python 3.7.2 (v3.7.2:9a3ffc0492, Dec 24 2018, 02:44:43)
Type "help", "copyright", "credits" or "license" for more information.
Welcome to
      ____              __
     / __/__  ___ _____/ /__
    _\ \/ _ \/ _ `/ __/  '_/
   /__ / .__/\_,_/_/ /_/\_\   version 2.4.0
      /_/

Using Python version 3.7.2 (v3.7.2:9a3ffc0492, Dec 24 2018 02:44:43)
SparkSession available as 'spark'.
>>> numbers = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
>>> rdd = sc.parallelize(numbers, 3)
>>> rdd.collect()
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
>>> rdd.count()
10
>>> rdd.getNumPartitions()
3
>>> def f(iterator):
...     for x in iterator:
...         print(x)
...     print("=====")
...
>>>
>>> rdd.foreachPartition(f)
4
5
6
=====
7
8
9
10
=====
1
2
3
=====
>>> rdd_default = sc.parallelize(numbers)
>>> rdd_default.collect()
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
>>> rdd.getNumPartitions()
3
>>> rdd_default.getNumPartitions()
8
>>> rdd.foreachPartition(f)
4
5
6
=====
1
2
3
=====
7
8
9
10
=====
>>> rdd_default.foreachPartition(f)
6
=====
7
=====
3
=====
2
=====
8
=====
4
5
=====
9
10
=====
1
=====
>>> numbers = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
>>> rdd_by_4 = sc.parallelize(numbers, 4)
>>> rdd_by_4.collect()
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
>>> rdd_by_4.foreachPartition(f)
1
2
3
=====
10
11
12
=====
4
5
6
=====
7
8
9
=====
>>> numbers = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15]
>>> rdd_by_6 = sc.parallelize(numbers, 6)
>>> rdd_by_6.foreachPartition(f)
7
8
=====
1
2
=====
11
12
13
15
=====
3
4
=====
9
10
=====
5
6
=====
>>> numbers = [1, 2, 3, 4, 5, 6]
>>> rdd_empty = sc.parallelize(numbers, 10)
>>> rdd_empty.foreachPartition(f)
2
=====
3
=====
=====
=====
4
=====
=====
6
=====
1
=====
5
=====
=====
>>>