Quick and dirty persons model
Historical people: Quick and dirty¶
This example shows how to get some initial record linkage results as quickly as possible.
There are many ways to improve the accuracy of this model. But this may be a good place to start if you just want to give Splink a try and see what it's capable of.
from splink.datasets import splink_datasets
from splink.internals.misc import show
df = splink_datasets.historical_50k
show(df, rows=5)
┌────────────┬──────────┬──────────────────────────────────────────────────┬───────────────────┬────────────┬───────────┬────────────┬─────────────┬───────────────┬─────────┬────────────┐
│ unique_id │ cluster │ full_name │ first_and_surname │ first_name │ surname │ dob │ birth_place │ postcode_fake │ gender │ occupation │
│ varchar │ varchar │ varchar │ varchar │ varchar │ varchar │ varchar │ varchar │ varchar │ varchar │ varchar │
├────────────┼──────────┼──────────────────────────────────────────────────┼───────────────────┼────────────┼───────────┼────────────┼─────────────┼───────────────┼─────────┼────────────┤
│ Q2296770-1 │ Q2296770 │ thomas clifford, 1st baron clifford of chudleigh │ thomas chudleigh │ thomas │ chudleigh │ 1630-08-01 │ devon │ tq13 8df │ male │ politician │
│ Q2296770-2 │ Q2296770 │ thomas of chudleigh │ thomas chudleigh │ thomas │ chudleigh │ 1630-08-01 │ devon │ tq13 8df │ male │ politician │
│ Q2296770-3 │ Q2296770 │ tom 1st baron clifford of chudleigh │ tom chudleigh │ tom │ chudleigh │ 1630-08-01 │ devon │ tq13 8df │ male │ politician │
│ Q2296770-4 │ Q2296770 │ thomas 1st chudleigh │ thomas chudleigh │ thomas │ chudleigh │ 1630-08-01 │ devon │ tq13 8hu │ NULL │ politician │
│ Q2296770-5 │ Q2296770 │ thomas clifford, 1st baron chudleigh │ thomas chudleigh │ thomas │ chudleigh │ 1630-08-01 │ devon │ tq13 8df │ NULL │ politician │
└────────────┴──────────┴──────────────────────────────────────────────────┴───────────────────┴────────────┴───────────┴────────────┴─────────────┴───────────────┴─────────┴────────────┘
from splink import block_on, SettingsCreator
import splink.comparison_library as cl
settings = SettingsCreator(
link_type="dedupe_only",
blocking_rules_to_generate_predictions=[
block_on("full_name"),
block_on("substr(full_name,1,6)", "dob", "birth_place"),
block_on("dob", "birth_place"),
block_on("postcode_fake"),
],
comparisons=[
cl.ForenameSurnameComparison(
"first_name",
"surname",
forename_surname_concat_col_name="first_and_surname",
),
cl.DateOfBirthComparison(
"dob",
input_is_string=True,
),
cl.LevenshteinAtThresholds("postcode_fake", 2),
cl.JaroWinklerAtThresholds("birth_place", 0.9).configure(
term_frequency_adjustments=True
),
cl.ExactMatch("occupation").configure(term_frequency_adjustments=True),
],
)
from splink import Linker, DuckDBAPI
db_api = DuckDBAPI()
df_sdf = db_api.register(df)
linker = Linker(df_sdf, settings, log_level=None)
deterministic_rules = [
"l.full_name = r.full_name",
"l.postcode_fake = r.postcode_fake and l.dob = r.dob",
]
linker.training.estimate_probability_two_random_records_match(
deterministic_rules, recall=0.6
)
linker.training.estimate_u_using_random_sampling(max_pairs=2e6)
results = linker.inference.predict(threshold_match_probability=0.9)
-- WARNING --
You have called predict(), but there are some parameter estimates which have neither been estimated or specified in your settings dictionary. To produce predictions the following untrained parameters will use default values.
Comparison: 'first_name_surname':
m values not fully trained
Comparison: 'first_name_surname':
u values not fully trained
Comparison: 'dob':
m values not fully trained
Comparison: 'postcode_fake':
m values not fully trained
Comparison: 'birth_place':
m values not fully trained
Comparison: 'occupation':
m values not fully trained
results.as_duckdbpyrelation().limit(5).show(max_width=10000)
┌────────────────────┬────────────────────┬─────────────┬─────────────┬──────────────┬──────────────┬───────────┬───────────┬─────────────────────┬─────────────────────┬──────────────────────────┬────────────┬────────────┬───────────┬─────────────────┬─────────────────┬─────────────────────┬───────────────┬───────────────┬───────────────────┬────────────────────┬────────────────┬──────────────────┬──────────────────┬─────────────┬───────────┐
│ match_weight │ match_probability │ unique_id_l │ unique_id_r │ first_name_l │ first_name_r │ surname_l │ surname_r │ first_and_surname_l │ first_and_surname_r │ gamma_first_name_surname │ dob_l │ dob_r │ gamma_dob │ postcode_fake_l │ postcode_fake_r │ gamma_postcode_fake │ birth_place_l │ birth_place_r │ gamma_birth_place │ occupation_l │ occupation_r │ gamma_occupation │ full_name_l │ full_name_r │ match_key │
│ double │ double │ varchar │ varchar │ varchar │ varchar │ varchar │ varchar │ varchar │ varchar │ int32 │ varchar │ varchar │ int32 │ varchar │ varchar │ int32 │ varchar │ varchar │ int32 │ varchar │ varchar │ int32 │ varchar │ varchar │ varchar │
├────────────────────┼────────────────────┼─────────────┼─────────────┼──────────────┼──────────────┼───────────┼───────────┼─────────────────────┼─────────────────────┼──────────────────────────┼────────────┼────────────┼───────────┼─────────────────┼─────────────────┼─────────────────────┼───────────────┼───────────────┼───────────────────┼────────────────────┼────────────────┼──────────────────┼──────────────────┼─────────────┼───────────┤
│ 16.784836267237672 │ 0.9999911436052675 │ Q15998133-3 │ Q15998133-9 │ horatio │ michell │ mitchell │ NULL │ horatio mitchell │ michell │ 0 │ 1858-02-09 │ 1858-02-09 │ 5 │ bn13 1dt │ bn13 1dt │ 2 │ tarring │ tarring │ 2 │ cricketer │ cricketer │ 1 │ horatio mitchell │ michell │ 2 │
│ 16.784836267237672 │ 0.9999911436052675 │ Q15998133-4 │ Q15998133-9 │ horace │ michell │ mitchell │ NULL │ horace mitchell │ michell │ 0 │ 1858-02-09 │ 1858-02-09 │ 5 │ bn13 1dt │ bn13 1dt │ 2 │ tarring │ tarring │ 2 │ cricketer │ cricketer │ 1 │ horace mitchell │ michell │ 2 │
│ 16.784836267237672 │ 0.9999911436052675 │ Q15998133-6 │ Q15998133-9 │ horatio │ michell │ mitchel │ NULL │ horatio mitchel │ michell │ 0 │ 1858-02-09 │ 1858-02-09 │ 5 │ bn13 1dt │ bn13 1dt │ 2 │ tarring │ tarring │ 2 │ cricketer │ cricketer │ 1 │ horatio mitchel │ michell │ 2 │
│ 9.716905264289107 │ 0.9988131268863198 │ Q6225085-1 │ Q6225085-6 │ john │ fapper │ capper │ NULL │ john capper │ fapper │ 0 │ 1861-12-07 │ 1861-12-07 │ 5 │ sy11 3pz │ sy11 3pz │ 2 │ lucknow │ lucknow │ 2 │ military personnel │ aircraft pilot │ 0 │ john capper │ fapper │ 2 │
│ 9.716905264289107 │ 0.9988131268863198 │ Q6225085-2 │ Q6225085-6 │ john │ fapper │ capper │ NULL │ john capper │ fapper │ 0 │ 1861-12-07 │ 1861-12-07 │ 5 │ sy11 3pz │ sy11 3pz │ 2 │ lucknow │ lucknow │ 2 │ military personnel │ aircraft pilot │ 0 │ john capper │ fapper │ 2 │
└────────────────────┴────────────────────┴─────────────┴─────────────┴──────────────┴──────────────┴───────────┴───────────┴─────────────────────┴─────────────────────┴──────────────────────────┴────────────┴────────────┴───────────┴─────────────────┴─────────────────┴─────────────────────┴───────────────┴───────────────┴───────────────────┴────────────────────┴────────────────┴──────────────────┴──────────────────┴─────────────┴───────────┘