Skip to content
Snippets Groups Projects
Commit 81230ac7 authored by Eric Pershey's avatar Eric Pershey
Browse files

fixing register_matcher.

parent c36e0546
No related branches found
No related tags found
No related merge requests found
......@@ -174,7 +174,8 @@ def register_matcher(spark: SparkSession, name, rules: Dict[str, str], delim: st
print("Not registering matcher from the jar file.")
else:
raise
rules_r = {cat: re.compile(regex) for cat, regex in rules.items()}
func_classify = functools.partial(line_matcher_regex, rules_r, delim)
udf_func_classify = F.udf(func_classify, returnType=StringType())
spark.udf.register(name, udf_func_classify)
print(f"Not registering matcher {name}, already registered.")
rules_r = {cat: re.compile(regex) for cat, regex in rules.items()}
func_classify = functools.partial(line_matcher_regex, rules_r, delim)
udf_func_classify = F.udf(func_classify, returnType=StringType())
spark.udf.register(name, udf_func_classify)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment