Quick and dirty persons model
Historical people: Quick and dirty¶
This example shows how to get some initial record linkage results as quickly as possible.
There are many ways to improve the accuracy of this model. But this may be a good place to start if you just want to give Splink a try and see what it's capable of.
from splink.datasets import splink_datasets
from splink.internals.misc import show
df = splink_datasets.historical_50k
show(df, rows=5)
┌────────────┬──────────┬──────────────────────────────────────────────────┬───────────────────┬────────────┬───────────┬────────────┬─────────────┬───────────────┬─────────┬────────────┐
│ unique_id │ cluster │ full_name │ first_and_surname │ first_name │ surname │ dob │ birth_place │ postcode_fake │ gender │ occupation │
│ varchar │ varchar │ varchar │ varchar │ varchar │ varchar │ varchar │ varchar │ varchar │ varchar │ varchar │
├────────────┼──────────┼──────────────────────────────────────────────────┼───────────────────┼────────────┼───────────┼────────────┼─────────────┼───────────────┼─────────┼────────────┤
│ Q2296770-1 │ Q2296770 │ thomas clifford, 1st baron clifford of chudleigh │ thomas chudleigh │ thomas │ chudleigh │ 1630-08-01 │ devon │ tq13 8df │ male │ politician │
│ Q2296770-2 │ Q2296770 │ thomas of chudleigh │ thomas chudleigh │ thomas │ chudleigh │ 1630-08-01 │ devon │ tq13 8df │ male │ politician │
│ Q2296770-3 │ Q2296770 │ tom 1st baron clifford of chudleigh │ tom chudleigh │ tom │ chudleigh │ 1630-08-01 │ devon │ tq13 8df │ male │ politician │
│ Q2296770-4 │ Q2296770 │ thomas 1st chudleigh │ thomas chudleigh │ thomas │ chudleigh │ 1630-08-01 │ devon │ tq13 8hu │ NULL │ politician │
│ Q2296770-5 │ Q2296770 │ thomas clifford, 1st baron chudleigh │ thomas chudleigh │ thomas │ chudleigh │ 1630-08-01 │ devon │ tq13 8df │ NULL │ politician │
└────────────┴──────────┴──────────────────────────────────────────────────┴───────────────────┴────────────┴───────────┴────────────┴─────────────┴───────────────┴─────────┴────────────┘
from splink import block_on, SettingsCreator
import splink.comparison_library as cl
settings = SettingsCreator(
link_type="dedupe_only",
blocking_rules_to_generate_predictions=[
block_on("full_name"),
block_on("substr(full_name,1,6)", "dob", "birth_place"),
block_on("dob", "birth_place"),
block_on("postcode_fake"),
],
comparisons=[
cl.ForenameSurnameComparison(
"first_name",
"surname",
forename_surname_concat_col_name="first_and_surname",
),
cl.DateOfBirthComparison(
"dob",
input_is_string=True,
),
cl.LevenshteinAtThresholds("postcode_fake", 2),
cl.JaroWinklerAtThresholds("birth_place", 0.9).configure(
term_frequency_adjustments=True
),
cl.ExactMatch("occupation").configure(term_frequency_adjustments=True),
],
)
from splink import Linker, DuckDBAPI
db_api = DuckDBAPI()
df_sdf = db_api.register(df)
linker = Linker(df_sdf, settings, log_level=None)
deterministic_rules = [
"l.full_name = r.full_name",
"l.postcode_fake = r.postcode_fake and l.dob = r.dob",
]
linker.training.estimate_probability_two_random_records_match(
deterministic_rules, recall=0.6
)
linker.training.estimate_u_using_random_sampling(max_pairs=2e6)
results = linker.inference.predict(threshold_match_probability=0.9)
-- WARNING --
You have called predict(), but there are some parameter estimates which have neither been estimated or specified in your settings dictionary. To produce predictions the following untrained parameters will use default values.
Comparison: 'first_name_surname':
m values not fully trained
Comparison: 'first_name_surname':
u values not fully trained
Comparison: 'dob':
m values not fully trained
Comparison: 'postcode_fake':
m values not fully trained
Comparison: 'birth_place':
m values not fully trained
Comparison: 'occupation':
m values not fully trained
results.as_duckdbpyrelation().limit(5).show(max_width=10000)
┌───────────────────┬────────────────────┬──────────────┬─────────────┬───────────┬───────────┬──────────────┬──────────────┬─────────────────────┬─────────────────────┬──────────────────────────┬────────────┬────────────┬───────────┬─────────────────┬─────────────────┬─────────────────────┬───────────────┬───────────────┬───────────────────┬─────────────────────────────┬──────────────┬──────────────────┬────────────────────┬────────────────┬───────────┐
│ match_weight │ match_probability │ unique_id_l │ unique_id_r │ surname_l │ surname_r │ first_name_l │ first_name_r │ first_and_surname_l │ first_and_surname_r │ gamma_first_name_surname │ dob_l │ dob_r │ gamma_dob │ postcode_fake_l │ postcode_fake_r │ gamma_postcode_fake │ birth_place_l │ birth_place_r │ gamma_birth_place │ occupation_l │ occupation_r │ gamma_occupation │ full_name_l │ full_name_r │ match_key │
│ double │ double │ varchar │ varchar │ varchar │ varchar │ varchar │ varchar │ varchar │ varchar │ int32 │ varchar │ varchar │ int32 │ varchar │ varchar │ int32 │ varchar │ varchar │ int32 │ varchar │ varchar │ int32 │ varchar │ varchar │ varchar │
├───────────────────┼────────────────────┼──────────────┼─────────────┼───────────┼───────────┼──────────────┼──────────────┼─────────────────────┼─────────────────────┼──────────────────────────┼────────────┼────────────┼───────────┼─────────────────┼─────────────────┼─────────────────────┼───────────────┼───────────────┼───────────────────┼─────────────────────────────┼──────────────┼──────────────────┼────────────────────┼────────────────┼───────────┤
│ 9.079344070771121 │ 0.9981548019194934 │ Q18810722-5 │ Q18810722-9 │ goulding │ NULL │ frederick │ goulding │ frederick goulding │ goulding │ 0 │ 1842-01-01 │ 1842-01-01 │ 5 │ n4 4at │ n4 4at │ 2 │ islington │ islington │ 2 │ printer │ NULL │ -1 │ frederick goulding │ goulding │ 2 │
│ 9.079344070771121 │ 0.9981548019194934 │ Q18810722-10 │ Q18810722-9 │ goulding │ NULL │ frederick │ goulding │ frederick goulding │ goulding │ 0 │ 1842-01-01 │ 1842-01-01 │ 5 │ n4 4at │ n4 4at │ 2 │ islington │ islington │ 2 │ printer │ NULL │ -1 │ frederick goulding │ goulding │ 2 │
│ 26.26233476921406 │ 0.9999999875763425 │ Q3806593-1 │ Q3806593-8 │ hamilton │ hamilton │ james │ jamie │ james hamilton │ jamie hamilton │ 3 │ 1859-01-01 │ 1859-01-01 │ 5 │ bt2 8lx │ bt2 8lx │ 2 │ belfast │ belfast │ 2 │ association football player │ NULL │ -1 │ james hamilton │ jamie hamilton │ 2 │
│ 26.26233476921406 │ 0.9999999875763425 │ Q3806593-2 │ Q3806593-8 │ hamilton │ hamilton │ james │ jamie │ james hamilton │ jamie hamilton │ 3 │ 1859-01-01 │ 1859-01-01 │ 5 │ bt2 8lx │ bt2 8lx │ 2 │ belfast │ belfast │ 2 │ association football player │ NULL │ -1 │ james hamilton │ jamie hamilton │ 2 │
│ 26.26233476921406 │ 0.9999999875763425 │ Q3806593-4 │ Q3806593-8 │ hamilton │ hamilton │ james │ jamie │ james hamilton │ jamie hamilton │ 3 │ 1859-01-01 │ 1859-01-01 │ 5 │ bt2 8lx │ bt2 8lx │ 2 │ belfast │ belfast │ 2 │ association football player │ NULL │ -1 │ james hamilton │ jamie hamilton │ 2 │
└───────────────────┴────────────────────┴──────────────┴─────────────┴───────────┴───────────┴──────────────┴──────────────┴─────────────────────┴─────────────────────┴──────────────────────────┴────────────┴────────────┴───────────┴─────────────────┴─────────────────┴─────────────────────┴───────────────┴───────────────┴───────────────────┴─────────────────────────────┴──────────────┴──────────────────┴────────────────────┴────────────────┴───────────┘