エラー:共用体は、互換性のある列タイプのテーブルでのみ実行できます。 struct(tier:string、skyward_number:string、skyward_points:string)<> struct(skyward_number:string、tier:string、skyward_points:string) 2番目のテーブルの最初の列;;
ここでは、構造体フィールドの順序は異なりますが、残りはすべて同じです。
dataframe1スキーマ
root
|-- emcg_uuid: string (nullable = true)
|-- name: string (nullable = true)
|-- phone_no: string (nullable = true)
|-- dob: string (nullable = true)
|-- country: string (nullable = true)
|-- travel_type: string (nullable = true)
|-- gdpr_restricted_flg: string (nullable = false)
|-- gdpr_reason_code: string (nullable = false)
|-- document: map (nullable = true)
| |-- key: string
| |-- value: string (valueContainsNull = true)
|-- skyward: struct (nullable = false)
| |-- tier: string (nullable = false)
| |-- skyward_number: string (nullable = false)
| |-- skyward_points: string (nullable = false)
dataframe2 schema
root
|-- emcg_uuid: string (nullable = true)
|-- name: string (nullable = true)
|-- phone_no: string (nullable = true)
|-- dob: string (nullable = true)
|-- country: string (nullable = true)
|-- travel_type: string (nullable = true)
|-- gdpr_restricted_flg: string (nullable = true)
|-- gdpr_reason_code: string (nullable = true)
|-- document: map (nullable = true)
| |-- key: string
| |-- value: string (valueContainsNull = true)
|-- skyward: struct (nullable = false)
| |-- skyward_number: string (nullable = false)
| |-- tier: string (nullable = false)
| |-- skyward_points: string (nullable = false)
これを解決する方法は?
//preserves the order the columns while doing union
def getStructRecursiveDataFrame(df1 : DataFrame, df2 : DataFrame,columns : Array[String]) : DataFrame = {
if(columns.isEmpty) {
df2
}
else {
println("test")
val col_name = columns.head
val col_schema = df1.schema.fields.find(_.name == col_name).get
if(col_schema.dataType.typeName.equals("struct")){
println("test1")
val updatedStructNames: Seq[Column] = col_schema.dataType.asInstanceOf[StructType].fieldNames.map(name => col(col_name+"." + name))
getStructRecursiveDataFrame(df1,df2.withColumn(col_name, struct(updatedStructNames: _*)),columns.tail)
}
else{ getStructRecursiveDataFrame(df1,df2,columns.tail)}
}
}
def unionByName(a: org.Apache.spark.sql.DataFrame, b: org.Apache.spark.sql.DataFrame): org.Apache.spark.sql.DataFrame = {
val b_new_df = getStructRecursiveDataFrame(a,b,a.columns)
val columns_seq = a.columns.toSet.intersect(b_new_df.columns.toSet).map(col).toSeq
a.select(columns_seq: _*).union(b_new_df.select(columns_seq: _*))
}
結果
[INFO] DATAFRAME-1 SCHEME
root
|-- emcg_uuid: string (nullable = true)
|-- name: string (nullable = true)
|-- phone_no: string (nullable = true)
|-- dob: string (nullable = true)
|-- country: string (nullable = true)
|-- travel_type: string (nullable = true)
|-- gdpr_restricted_flg: string (nullable = false)
|-- gdpr_reason_code: string (nullable = false)
|-- document: map (nullable = true)
| |-- key: string
| |-- value: string (valueContainsNull = true)
|-- skyward: struct (nullable = false)
| |-- tier: string (nullable = false)
| |-- skyward_number: string (nullable = false)
| |-- skyward_points: string (nullable = false)
[INFO] DATAFRAME-2 SCHEME
root
|-- emcg_uuid: string (nullable = true)
|-- name: string (nullable = true)
|-- phone_no: string (nullable = true)
|-- dob: string (nullable = true)
|-- country: string (nullable = true)
|-- travel_type: string (nullable = true)
|-- gdpr_restricted_flg: string (nullable = true)
|-- gdpr_reason_code: string (nullable = true)
|-- document: map (nullable = true)
| |-- key: string
| |-- value: string (valueContainsNull = true)
|-- skyward: struct (nullable = false)
| |-- skyward_number: string (nullable = false)
| |-- tier: string (nullable = false)
| |-- skyward_points: string (nullable = false)
[INFO] DATAFRAME SCHEME AFTER THE UNION
root
|-- skyward: struct (nullable = false)
| |-- skyward_number: string (nullable = false)
| |-- tier: string (nullable = false)
| |-- skyward_points: string (nullable = false)
|-- name: string (nullable = true)
|-- document: map (nullable = true)
| |-- key: string
| |-- value: string (valueContainsNull = true)
|-- phone_no: string (nullable = true)
|-- travel_type: string (nullable = true)
|-- gdpr_restricted_flg: string (nullable = true)
|-- dob: string (nullable = true)
|-- gdpr_reason_code: string (nullable = true)
|-- country: string (nullable = true)
|-- emcg_uuid: string (nullable = true)
[INFO] TEST CASE FOR ANONYMIZATION VALIDATION
[INFO] INPUT DATA
+----+----------+-----------+-------------------+----------+----------------+-------+---------+-------------------------------------------+-----------------+
|name|phone_no |travel_type|gdpr_restricted_flg|dob |gdpr_reason_code|country|emcg_uuid|document |skyward |
+----+----------+-----------+-------------------+----------+----------------+-------+---------+-------------------------------------------+-----------------+
|ravi|8747436090|freq | |1988-05-28| |dubai |uuid_1 |Map(document_type -> passport, id -> A3343)|[123456,blue,687]|
|aaaa|8747436091|freg | |1988-06-25| |europe |uuid_2 |Map(document_type -> passport, id -> A3341)|[123456,blue,687]|
|bbbb|8747436092|reg | |1988-07-26| |india |uuid_3 |Map(document_type -> passport, id -> A3345)|[123456,blue,687]|
|cccc|8747436093|na | |1988-08-27| |georgia|uuid_4 |Map(document_type -> passport, id -> A3349)|[123456,blue,687]|
|dddd|8747436094|na | |1988-09-29| |swis |uuid_5 |Map(document_type -> passport, id -> B3343)|[123456,blue,687]|
|null|8747436095|freq | |1988-02-30| |us |uuid_6 |Map(document_type -> passport, id -> C3343)|[123456,blue,687]|
|null|8747436096|na | |1988-01-01| |canada |uuid_7 |Map(document_type -> null, id -> D3343) |[123456,blue,687]|
+----+----------+-----------+-------------------+----------+----------------+-------+---------+-------------------------------------------+-----------------+
[INFO] EXPECTED OUTPUT
+-------+----------+-----------+-------------------+----------+----------------+-------+---------+-------------------------------------------+-----------------+
|name |phone_no |travel_type|gdpr_restricted_flg|dob |gdpr_reason_code|country|emcg_uuid|document |skyward |
+-------+----------+-----------+-------------------+----------+----------------+-------+---------+-------------------------------------------+-----------------+
|DDDDDDD|9999999 |freq |Y |1988-05-XX|13-001 |XXXXXXX|uuid_1 |Map(document_type -> ZZZZZ, id -> HH343) |[123456,blue,687]|
|aaaa |8747436091|freg | |1988-06-25| |europe |uuid_2 |Map(document_type -> passport, id -> A3341)|[123456,blue,687]|
|DDDDDDD|9999999 |reg |Y |1988-07-XX|13-001 |XXXXXXX|uuid_3 |Map(document_type -> ZZZZZ, id -> HH345) |[123456,blue,687]|
|cccc |8747436093|na | |1988-08-27| |georgia|uuid_4 |Map(document_type -> passport, id -> A3349)|[123456,blue,687]|
|dddd |8747436094|na | |1988-09-29| |swis |uuid_5 |Map(document_type -> passport, id -> B3343)|[123456,blue,687]|
|null |8747436095|freq | |1988-02-30| |us |uuid_6 |Map(document_type -> passport, id -> C3343)|[123456,blue,687]|
|null |9999999 |na |Y |1988-01-XX|13-001 |XXXXXXX|uuid_7 |Map(document_type -> null, id -> HH343) |[123456,blue,687]|
+-------+----------+-----------+-------------------+----------+----------------+-------+---------+-------------------------------------------+-----------------+
[INFO] ACTUAL OUTPUT
+-------+----------+-----------+-------------------+----------+----------------+-------+---------+-------------------------------------------+-----------------+
|name |phone_no |travel_type|gdpr_restricted_flg|dob |gdpr_reason_code|country|emcg_uuid|document |skyward |
+-------+----------+-----------+-------------------+----------+----------------+-------+---------+-------------------------------------------+-----------------+
|DDDDDDD|9999999 |freq |Y |1988-05-XX|13-001 |XXXXXXX|uuid_1 |Map(document_type -> ZZZZZ, id -> HH343) |[UUUUU,blue,JJ7] |
|aaaa |8747436091|freg | |1988-06-25| |europe |uuid_2 |Map(document_type -> passport, id -> A3341)|[123456,blue,687]|
|DDDDDDD|9999999 |reg |Y |1988-07-XX|13-001 |XXXXXXX|uuid_3 |Map(document_type -> ZZZZZ, id -> HH345) |[UUUUU,blue,JJ7] |
|cccc |8747436093|na | |1988-08-27| |georgia|uuid_4 |Map(document_type -> passport, id -> A3349)|[123456,blue,687]|
|dddd |8747436094|na | |1988-09-29| |swis |uuid_5 |Map(document_type -> passport, id -> B3343)|[123456,blue,687]|
|null |8747436095|freq | |1988-02-30| |us |uuid_6 |Map(document_type -> passport, id -> C3343)|[123456,blue,687]|
|null |9999999 |na |Y |1988-01-XX|13-001 |XXXXXXX|uuid_7 |Map(document_type -> null, id -> HH343) |[UUUUU,blue,JJ7] |
+-------+----------+-----------+-------------------+----------+----------------+-------+---------+-------------------------------------------+-----------------+
1つのフィールドのみが異なり、名前がわかっている場合( "skyward")、次のように解決できます。
val data = List(("1", "2", "3"))
val bulkDF = data.toDF("emcg_uuid", "tier", "skyward_number")
// union parts
val tsDF = bulkDF.withColumn("skyward", struct($"tier", $"skyward_number"))
val stDF = bulkDF.withColumn("skyward", struct($"skyward_number", $"tier"))
// change struct "skyward" in last stDF
val schema = tsDF.schema.fields.find(_.name == "skyward").get
val updatedStructNames: Seq[Column] = schema.dataType.asInstanceOf[StructType].fieldNames.map(name => col("skyward." + name))
val withUpdatedSchema = stDF.withColumn("skyward", struct(updatedStructNames: _*))
// union
tsDF.union(withUpdatedSchema).show(false)
このような構造体フィールドの多くでは、いくつかのループを使用できます。