Quick and dirty persons model

Historical people: Quick and dirty

This example shows how to get some initial record linkage results as quickly as possible.

There are many ways to improve the accuracy of this model. But this may be a good place to start if you just want to give Splink a try and see what it's capable of.

from splink.datasets import splink_datasets
df = splink_datasets.historical_50k
df.head(5)

	unique_id	cluster	full_name	first_and_surname	first_name	surname	dob	birth_place	postcode_fake	gender	occupation
0	Q2296770-1	Q2296770	thomas clifford, 1st baron clifford of chudleigh	thomas chudleigh	thomas	chudleigh	1630-08-01	devon	tq13 8df	male	politician
1	Q2296770-2	Q2296770	thomas of chudleigh	thomas chudleigh	thomas	chudleigh	1630-08-01	devon	tq13 8df	male	politician
2	Q2296770-3	Q2296770	tom 1st baron clifford of chudleigh	tom chudleigh	tom	chudleigh	1630-08-01	devon	tq13 8df	male	politician
3	Q2296770-4	Q2296770	thomas 1st chudleigh	thomas chudleigh	thomas	chudleigh	1630-08-01	devon	tq13 8hu	None	politician
4	Q2296770-5	Q2296770	thomas clifford, 1st baron chudleigh	thomas chudleigh	thomas	chudleigh	1630-08-01	devon	tq13 8df	None	politician

from splink.duckdb.linker import DuckDBLinker
from splink.duckdb.blocking_rule_library import block_on
import splink.duckdb.comparison_library as cl

settings = {
    "link_type": "dedupe_only",
    "blocking_rules_to_generate_predictions": [
        block_on("full_name"),
        block_on(["substr(full_name,1,6)", "dob", "birth_place"]),
        block_on(["dob", "birth_place"]),
        block_on("postcode_fake"),
    ],
    "comparisons": [
        cl.jaro_at_thresholds("full_name", [0.9, 0.7], term_frequency_adjustments=True),
        cl.levenshtein_at_thresholds("dob", [1, 2]),
        cl.levenshtein_at_thresholds("postcode_fake", 2),
        cl.jaro_winkler_at_thresholds("birth_place", 0.9, term_frequency_adjustments=True),
        cl.exact_match("occupation",  term_frequency_adjustments=True),
    ],       

}

linker = DuckDBLinker(df, settings, set_up_basic_logging=False)
deterministic_rules = [
    "l.full_name = r.full_name",
    "l.postcode_fake = r.postcode_fake and l.dob = r.dob",
]

linker.estimate_probability_two_random_records_match(deterministic_rules, recall=0.6)

linker.estimate_u_using_random_sampling(max_pairs=2e6)

results = linker.predict(threshold_match_probability=0.9)

 -- WARNING --
You have called predict(), but there are some parameter estimates which have neither been estimated or specified in your settings dictionary.  To produce predictions the following untrained trained parameters will use default values.
Comparison: 'full_name':
    m values not fully trained
Comparison: 'dob':
    m values not fully trained
Comparison: 'postcode_fake':
    m values not fully trained
Comparison: 'birth_place':
    m values not fully trained
Comparison: 'occupation':
    m values not fully trained

results.as_pandas_dataframe(limit=5)

	match_weight	match_probability	unique_id_l	unique_id_r	full_name_l	full_name_r	gamma_full_name	dob_l	dob_r	gamma_dob	postcode_fake_l	postcode_fake_r	gamma_postcode_fake	birth_place_l	birth_place_r	gamma_birth_place	occupation_l	occupation_r	gamma_occupation	match_key
0	33.962763	1.000000	Q90404618-1	Q90404618-3	emlie clifford	emlie clifford	3	1861-01-01	1861-01-01	3	wr11 7qp	wr11 7qw	1	wychavon	wychavon	2	playwright	playwright	1	0
1	33.962763	1.000000	Q90404618-2	Q90404618-3	emlie clifford	emlie clifford	3	1861-01-01	1861-01-01	3	wr11 7qp	wr11 7qw	1	wychavon	wychavon	2	playwright	playwright	1	0
2	16.224687	0.999987	Q55455287-1	Q55455287-8	jaido morata	jaido morata	3	1836-01-01	1836-11-01	2	ta4 2uu	ta4 2uu	2	somerset west and taunton	NaN	-1	writer	writer	1	0
3	16.224687	0.999987	Q55455287-2	Q55455287-8	jaido morata	jaido morata	3	1836-01-01	1836-11-01	2	ta4 2uu	ta4 2uu	2	somerset west and taunton	NaN	-1	writer	writer	1	0
4	16.224687	0.999987	Q55455287-3	Q55455287-8	jaido morata	jaido morata	3	1836-01-01	1836-11-01	2	ta4 2uu	ta4 2uu	2	somerset west and taunton	NaN	-1	writer	writer	1	0