concat = pd.merge(data_1, data_2, how='inner')

import pandas as pd

​

# Drop all duplicates in the DataFrame

df = df.drop_duplicates()

​

# Drop all duplicates in a specific column of the DataFrame

df = df.drop_duplicates(subset = "column")

​

# Drop all duplicate pairs in DataFrame

df = df.drop_duplicates(subset = ["column", "column2"])

​

# Display DataFrame

print(df)

df.drop_duplicates(['A','B'],keep= 'last')

# Remove by index

df = df[df.index.duplicated(keep='first')]

​

# Other methods to remove duplicates

import pandas as pd

​

df = df.drop_duplicates()

​

df = df.drop_duplicates(subset = "column")

​

df = df.drop_duplicates(subset = ["column", "column2"])

df = df.loc[:,~df.columns.duplicated()]

# If same dataset needs to be updated:

​

df.drop_duplicates(keep=False, inplace=True)

#Create test data

df1 = pd.DataFrame(np.random.randint(100,size=(1000, 3)),columns=['A','B','C'])

df2 = pd.DataFrame(np.random.randint(100,size=(1000, 3)),columns=['B','C','D'])

​

pd.merge(df1, df2, how='inner', left_on=['B','C'], right_on=['B','C'])

df = df.loc[:,~df.columns.duplicated()].copy()

​

# https://stackoverflow.com/questions/14984119/python-pandas-remove-duplicate-columns

import pandas as pd

import numpy as np

​

data = {'column1': [1, 2, 2, np.nan, 4],

        'column2': ['a', 'b', 'b', 'c', 'd']}

​

df1 = df2 = df3 = pd.DataFrame(data)

​

# Remove all duplicates inplace

df1.drop_duplicates(inplace=True)

print(df1,"\n")

​

# Remove duplicates in one column inplace

df2.drop_duplicates(subset="column1", inplace=True)

print(df2,"\n")

​

# Remove duplicates in multiple columns inplace

df3.drop_duplicates(subset=["column1", "column2"], inplace=True)

print(df3)