feat: remove duplicate lines
This commit is contained in:
Ibrahima G. Coulibaly
@@ -1,3 +1,4 @@
|
||||
import { tool as stringRemoveDuplicateLines } from './remove-duplicate-lines/meta';
|
||||
import { tool as stringReverse } from './reverse/meta';
|
||||
import { tool as stringRandomizeCase } from './randomize-case/meta';
|
||||
import { tool as stringUppercase } from './uppercase/meta';
|
||||
@@ -11,6 +12,7 @@ import { tool as stringJoin } from './join/meta';
|
||||
export const stringTools = [
|
||||
stringSplit,
|
||||
stringJoin,
|
||||
stringRemoveDuplicateLines,
|
||||
stringToMorse
|
||||
// stringReverse,
|
||||
// stringRandomizeCase,
|
||||
|
||||
@@ -0,0 +1,260 @@
|
||||
import { Box } from '@mui/material';
|
||||
import React, { useRef, useState } from 'react';
|
||||
import ToolTextInput from '@components/input/ToolTextInput';
|
||||
import ToolTextResult from '@components/result/ToolTextResult';
|
||||
import ToolOptions, { GetGroupsType } from '@components/options/ToolOptions';
|
||||
import SimpleRadio from '@components/options/SimpleRadio';
|
||||
import CheckboxWithDesc from '@components/options/CheckboxWithDesc';
|
||||
import ToolInputAndResult from '@components/ToolInputAndResult';
|
||||
import ToolExamples, {
|
||||
CardExampleType
|
||||
} from '@components/examples/ToolExamples';
|
||||
import { ToolComponentProps } from '@tools/defineTool';
|
||||
import { FormikProps } from 'formik';
|
||||
import removeDuplicateLines, {
|
||||
DuplicateRemovalMode,
|
||||
DuplicateRemoverOptions,
|
||||
NewlineOption
|
||||
} from './service';
|
||||
|
||||
// Initial values for our form
|
||||
const initialValues: DuplicateRemoverOptions = {
|
||||
mode: 'all',
|
||||
newlines: 'filter',
|
||||
sortLines: false,
|
||||
trimTextLines: false
|
||||
};
|
||||
|
||||
// Operation mode options
|
||||
const operationModes = [
|
||||
{
|
||||
title: 'Remove All Duplicate Lines',
|
||||
description:
|
||||
'If this option is selected, then all repeated lines across entire text are removed, starting from the second occurrence.',
|
||||
value: 'all' as DuplicateRemovalMode
|
||||
},
|
||||
{
|
||||
title: 'Remove Consecutive Duplicate Lines',
|
||||
description:
|
||||
'If this option is selected, then only consecutive repeated lines are removed.',
|
||||
value: 'consecutive' as DuplicateRemovalMode
|
||||
},
|
||||
{
|
||||
title: 'Leave Absolutely Unique Text Lines',
|
||||
description:
|
||||
'If this option is selected, then all lines that appear more than once are removed.',
|
||||
value: 'unique' as DuplicateRemovalMode
|
||||
}
|
||||
];
|
||||
|
||||
// Newlines options
|
||||
const newlineOptions = [
|
||||
{
|
||||
title: 'Preserve All Newlines',
|
||||
description: 'Leave all empty lines in the output.',
|
||||
value: 'preserve' as NewlineOption
|
||||
},
|
||||
{
|
||||
title: 'Filter All Newlines',
|
||||
description: 'Process newlines as regular lines.',
|
||||
value: 'filter' as NewlineOption
|
||||
},
|
||||
{
|
||||
title: 'Delete All Newlines',
|
||||
description: 'Before filtering uniques, remove all newlines.',
|
||||
value: 'delete' as NewlineOption
|
||||
}
|
||||
];
|
||||
|
||||
// Example cards for demonstration
|
||||
const exampleCards: CardExampleType<typeof initialValues>[] = [
|
||||
{
|
||||
title: 'Remove Duplicate Items from List',
|
||||
description:
|
||||
'Removes duplicate items from a shopping list, keeping only the first occurrence of each item.',
|
||||
sampleText: `Apples
|
||||
Bananas
|
||||
Milk
|
||||
Eggs
|
||||
Bread
|
||||
Milk
|
||||
Cheese
|
||||
Apples
|
||||
Yogurt`,
|
||||
sampleResult: `Apples
|
||||
Bananas
|
||||
Milk
|
||||
Eggs
|
||||
Bread
|
||||
Cheese
|
||||
Yogurt`,
|
||||
sampleOptions: {
|
||||
...initialValues,
|
||||
mode: 'all',
|
||||
newlines: 'filter',
|
||||
sortLines: false,
|
||||
trimTextLines: false
|
||||
}
|
||||
},
|
||||
{
|
||||
title: 'Clean Consecutive Duplicates',
|
||||
description:
|
||||
'Removes consecutive duplicates from log entries, which often happen when a system repeatedly logs the same error.',
|
||||
sampleText: `[INFO] Application started
|
||||
[ERROR] Connection failed
|
||||
[ERROR] Connection failed
|
||||
[ERROR] Connection failed
|
||||
[INFO] Retrying connection
|
||||
[ERROR] Authentication error
|
||||
[ERROR] Authentication error
|
||||
[INFO] Connection established`,
|
||||
sampleResult: `[INFO] Application started
|
||||
[ERROR] Connection failed
|
||||
[INFO] Retrying connection
|
||||
[ERROR] Authentication error
|
||||
[INFO] Connection established`,
|
||||
sampleOptions: {
|
||||
...initialValues,
|
||||
mode: 'consecutive',
|
||||
newlines: 'filter',
|
||||
sortLines: false,
|
||||
trimTextLines: false
|
||||
}
|
||||
},
|
||||
{
|
||||
title: 'Extract Unique Entries Only',
|
||||
description:
|
||||
'Filters a list to keep only entries that appear exactly once, removing any duplicated items entirely.',
|
||||
sampleText: `Red
|
||||
Blue
|
||||
Green
|
||||
Blue
|
||||
Yellow
|
||||
Purple
|
||||
Red
|
||||
Orange`,
|
||||
sampleResult: `Green
|
||||
Yellow
|
||||
Purple
|
||||
Orange`,
|
||||
sampleOptions: {
|
||||
...initialValues,
|
||||
mode: 'unique',
|
||||
newlines: 'filter',
|
||||
sortLines: false,
|
||||
trimTextLines: false
|
||||
}
|
||||
},
|
||||
{
|
||||
title: 'Sort and Clean Data',
|
||||
description:
|
||||
'Removes duplicate items from a list, trims whitespace, and sorts the results alphabetically.',
|
||||
sampleText: ` Apple
|
||||
Banana
|
||||
Cherry
|
||||
Apple
|
||||
Banana
|
||||
Dragonfruit
|
||||
Elderberry `,
|
||||
sampleResult: `Apple
|
||||
Banana
|
||||
Cherry
|
||||
Dragonfruit
|
||||
Elderberry`,
|
||||
sampleOptions: {
|
||||
...initialValues,
|
||||
mode: 'all',
|
||||
newlines: 'filter',
|
||||
sortLines: true,
|
||||
trimTextLines: true
|
||||
}
|
||||
}
|
||||
];
|
||||
|
||||
export default function RemoveDuplicateLines({ title }: ToolComponentProps) {
|
||||
const [input, setInput] = useState<string>('');
|
||||
const [result, setResult] = useState<string>('');
|
||||
const formRef = useRef<FormikProps<typeof initialValues>>(null);
|
||||
|
||||
const computeExternal = (
|
||||
optionsValues: typeof initialValues,
|
||||
inputText: string
|
||||
) => {
|
||||
setResult(removeDuplicateLines(inputText, optionsValues));
|
||||
};
|
||||
|
||||
const getGroups: GetGroupsType<typeof initialValues> = ({
|
||||
values,
|
||||
updateField
|
||||
}) => [
|
||||
{
|
||||
title: 'Operation Mode',
|
||||
component: operationModes.map(({ title, description, value }) => (
|
||||
<SimpleRadio
|
||||
key={value}
|
||||
checked={value === values.mode}
|
||||
title={title}
|
||||
description={description}
|
||||
onClick={() => updateField('mode', value)}
|
||||
/>
|
||||
))
|
||||
},
|
||||
{
|
||||
title: 'Newlines, Tabs and Spaces',
|
||||
component: [
|
||||
...newlineOptions.map(({ title, description, value }) => (
|
||||
<SimpleRadio
|
||||
key={value}
|
||||
checked={value === values.newlines}
|
||||
title={title}
|
||||
description={description}
|
||||
onClick={() => updateField('newlines', value)}
|
||||
/>
|
||||
)),
|
||||
<CheckboxWithDesc
|
||||
key="trimTextLines"
|
||||
checked={values.trimTextLines}
|
||||
title="Trim Text Lines"
|
||||
description="Before filtering uniques, remove tabs and spaces from the beginning and end of all lines."
|
||||
onChange={(checked) => updateField('trimTextLines', checked)}
|
||||
/>
|
||||
]
|
||||
},
|
||||
{
|
||||
title: 'Sort Lines',
|
||||
component: [
|
||||
<CheckboxWithDesc
|
||||
key="sortLines"
|
||||
checked={values.sortLines}
|
||||
title="Sort the Output Lines"
|
||||
description="After removing the duplicates, sort the unique lines."
|
||||
onChange={(checked) => updateField('sortLines', checked)}
|
||||
/>
|
||||
]
|
||||
}
|
||||
];
|
||||
|
||||
return (
|
||||
<Box>
|
||||
<ToolInputAndResult
|
||||
input={<ToolTextInput value={input} onChange={setInput} />}
|
||||
result={
|
||||
<ToolTextResult title={'Text without duplicates'} value={result} />
|
||||
}
|
||||
/>
|
||||
<ToolOptions
|
||||
compute={computeExternal}
|
||||
getGroups={getGroups}
|
||||
initialValues={initialValues}
|
||||
input={input}
|
||||
/>
|
||||
<ToolExamples
|
||||
title={title}
|
||||
exampleCards={exampleCards}
|
||||
getGroups={getGroups}
|
||||
formRef={formRef}
|
||||
setInput={setInput}
|
||||
/>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { defineTool } from '@tools/defineTool';
|
||||
import { lazy } from 'react';
|
||||
|
||||
export const tool = defineTool('string', {
|
||||
name: 'Remove duplicate lines',
|
||||
path: 'remove-duplicate-lines',
|
||||
icon: 'pepicons-print:duplicate-off',
|
||||
description:
|
||||
"Load your text in the input form on the left and you'll instantly get text with no duplicate lines in the output area. Powerful, free, and fast. Load text lines – get unique text lines",
|
||||
shortDescription: 'Quickly delete all repeated lines from text',
|
||||
keywords: ['remove', 'duplicate', 'lines'],
|
||||
component: lazy(() => import('./index'))
|
||||
});
|
||||
@@ -0,0 +1,200 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import removeDuplicateLines, { DuplicateRemoverOptions } from './service';
|
||||
|
||||
describe('removeDuplicateLines function', () => {
|
||||
// Test for 'all' duplicate removal mode
|
||||
describe('mode: all', () => {
|
||||
it('should remove all duplicates keeping first occurrence', () => {
|
||||
const input = 'line1\nline2\nline1\nline3\nline2';
|
||||
const options: DuplicateRemoverOptions = {
|
||||
mode: 'all',
|
||||
newlines: 'filter',
|
||||
sortLines: false,
|
||||
trimTextLines: false
|
||||
};
|
||||
const result = removeDuplicateLines(input, options);
|
||||
expect(result).toBe('line1\nline2\nline3');
|
||||
});
|
||||
|
||||
it('should handle case-sensitive duplicates correctly', () => {
|
||||
const input = 'Line1\nline1\nLine2\nline2';
|
||||
const options: DuplicateRemoverOptions = {
|
||||
mode: 'all',
|
||||
newlines: 'filter',
|
||||
sortLines: false,
|
||||
trimTextLines: false
|
||||
};
|
||||
const result = removeDuplicateLines(input, options);
|
||||
expect(result).toBe('Line1\nline1\nLine2\nline2');
|
||||
});
|
||||
});
|
||||
|
||||
// Test for 'consecutive' duplicate removal mode
|
||||
describe('mode: consecutive', () => {
|
||||
it('should remove only consecutive duplicates', () => {
|
||||
const input = 'line1\nline1\nline2\nline3\nline3\nline1';
|
||||
const options: DuplicateRemoverOptions = {
|
||||
mode: 'consecutive',
|
||||
newlines: 'filter',
|
||||
sortLines: false,
|
||||
trimTextLines: false
|
||||
};
|
||||
const result = removeDuplicateLines(input, options);
|
||||
expect(result).toBe('line1\nline2\nline3\nline1');
|
||||
});
|
||||
});
|
||||
|
||||
// Test for 'unique' duplicate removal mode
|
||||
describe('mode: unique', () => {
|
||||
it('should keep only lines that appear exactly once', () => {
|
||||
const input = 'line1\nline2\nline1\nline3\nline4\nline4';
|
||||
const options: DuplicateRemoverOptions = {
|
||||
mode: 'unique',
|
||||
newlines: 'filter',
|
||||
sortLines: false,
|
||||
trimTextLines: false
|
||||
};
|
||||
const result = removeDuplicateLines(input, options);
|
||||
expect(result).toBe('line2\nline3');
|
||||
});
|
||||
});
|
||||
|
||||
// Test for newlines handling
|
||||
describe('newlines option', () => {
|
||||
it('should filter newlines when newlines is set to filter', () => {
|
||||
const input = 'line1\n\nline2\n\n\nline3';
|
||||
const options: DuplicateRemoverOptions = {
|
||||
mode: 'all',
|
||||
newlines: 'filter',
|
||||
sortLines: false,
|
||||
trimTextLines: false
|
||||
};
|
||||
const result = removeDuplicateLines(input, options);
|
||||
expect(result).toBe('line1\n\nline2\nline3');
|
||||
});
|
||||
|
||||
it('should delete newlines when newlines is set to delete', () => {
|
||||
const input = 'line1\n\nline2\n\n\nline3';
|
||||
const options: DuplicateRemoverOptions = {
|
||||
mode: 'all',
|
||||
newlines: 'delete',
|
||||
sortLines: false,
|
||||
trimTextLines: false
|
||||
};
|
||||
const result = removeDuplicateLines(input, options);
|
||||
expect(result).toBe('line1\nline2\nline3');
|
||||
});
|
||||
|
||||
it('should preserve newlines when newlines is set to preserve', () => {
|
||||
const input = 'line1\n\nline2\n\nline2\nline3';
|
||||
const options: DuplicateRemoverOptions = {
|
||||
mode: 'all',
|
||||
newlines: 'preserve',
|
||||
sortLines: false,
|
||||
trimTextLines: false
|
||||
};
|
||||
const result = removeDuplicateLines(input, options);
|
||||
// This test needs careful consideration of the expected behavior
|
||||
expect(result).not.toContain('line2\nline2');
|
||||
expect(result).toContain('line1');
|
||||
expect(result).toContain('line2');
|
||||
expect(result).toContain('line3');
|
||||
});
|
||||
});
|
||||
|
||||
// Test for sorting
|
||||
describe('sortLines option', () => {
|
||||
it('should sort lines when sortLines is true', () => {
|
||||
const input = 'line3\nline1\nline2';
|
||||
const options: DuplicateRemoverOptions = {
|
||||
mode: 'all',
|
||||
newlines: 'filter',
|
||||
sortLines: true,
|
||||
trimTextLines: false
|
||||
};
|
||||
const result = removeDuplicateLines(input, options);
|
||||
expect(result).toBe('line1\nline2\nline3');
|
||||
});
|
||||
});
|
||||
|
||||
// Test for trimming
|
||||
describe('trimTextLines option', () => {
|
||||
it('should trim lines when trimTextLines is true', () => {
|
||||
const input = ' line1 \n line2 \nline3';
|
||||
const options: DuplicateRemoverOptions = {
|
||||
mode: 'all',
|
||||
newlines: 'filter',
|
||||
sortLines: false,
|
||||
trimTextLines: true
|
||||
};
|
||||
const result = removeDuplicateLines(input, options);
|
||||
expect(result).toBe('line1\nline2\nline3');
|
||||
});
|
||||
|
||||
it('should consider trimmed lines as duplicates', () => {
|
||||
const input = ' line1 \nline1\n line2\nline2 ';
|
||||
const options: DuplicateRemoverOptions = {
|
||||
mode: 'all',
|
||||
newlines: 'filter',
|
||||
sortLines: false,
|
||||
trimTextLines: true
|
||||
};
|
||||
const result = removeDuplicateLines(input, options);
|
||||
expect(result).toBe('line1\nline2');
|
||||
});
|
||||
});
|
||||
|
||||
// Combined scenarios
|
||||
describe('combined options', () => {
|
||||
it('should handle all options together correctly', () => {
|
||||
const input = ' line3 \nline1\n\nline3\nline2\nline1';
|
||||
const options: DuplicateRemoverOptions = {
|
||||
mode: 'all',
|
||||
newlines: 'delete',
|
||||
sortLines: true,
|
||||
trimTextLines: true
|
||||
};
|
||||
const result = removeDuplicateLines(input, options);
|
||||
expect(result).toBe('line1\nline2\nline3');
|
||||
});
|
||||
});
|
||||
|
||||
// Edge cases
|
||||
describe('edge cases', () => {
|
||||
it('should handle empty input', () => {
|
||||
const input = '';
|
||||
const options: DuplicateRemoverOptions = {
|
||||
mode: 'all',
|
||||
newlines: 'filter',
|
||||
sortLines: false,
|
||||
trimTextLines: false
|
||||
};
|
||||
const result = removeDuplicateLines(input, options);
|
||||
expect(result).toBe('');
|
||||
});
|
||||
|
||||
it('should handle input with only newlines', () => {
|
||||
const input = '\n\n\n';
|
||||
const options: DuplicateRemoverOptions = {
|
||||
mode: 'all',
|
||||
newlines: 'filter',
|
||||
sortLines: false,
|
||||
trimTextLines: false
|
||||
};
|
||||
const result = removeDuplicateLines(input, options);
|
||||
expect(result).toBe('');
|
||||
});
|
||||
|
||||
it('should handle input with only whitespace', () => {
|
||||
const input = ' \n \n ';
|
||||
const options: DuplicateRemoverOptions = {
|
||||
mode: 'all',
|
||||
newlines: 'filter',
|
||||
sortLines: false,
|
||||
trimTextLines: true
|
||||
};
|
||||
const result = removeDuplicateLines(input, options);
|
||||
expect(result).toBe('');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,88 @@
|
||||
export type NewlineOption = 'preserve' | 'filter' | 'delete';
|
||||
export type DuplicateRemovalMode = 'all' | 'consecutive' | 'unique';
|
||||
|
||||
export interface DuplicateRemoverOptions {
|
||||
mode: DuplicateRemovalMode;
|
||||
newlines: NewlineOption;
|
||||
sortLines: boolean;
|
||||
trimTextLines: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes duplicate lines from text based on specified options
|
||||
* @param text The input text to process
|
||||
* @param options Configuration options for text processing
|
||||
* @returns Processed text with duplicates removed according to options
|
||||
*/
|
||||
export default function removeDuplicateLines(
|
||||
text: string,
|
||||
options: DuplicateRemoverOptions
|
||||
): string {
|
||||
// Split the text into individual lines
|
||||
let lines = text.split('\n');
|
||||
|
||||
// Process newlines based on option
|
||||
if (options.newlines === 'delete') {
|
||||
// Remove all empty lines
|
||||
lines = lines.filter((line) => line.trim() !== '');
|
||||
}
|
||||
|
||||
// Trim lines if option is selected
|
||||
if (options.trimTextLines) {
|
||||
lines = lines.map((line) => line.trim());
|
||||
}
|
||||
|
||||
// Remove duplicates based on mode
|
||||
let processedLines: string[] = [];
|
||||
|
||||
if (options.mode === 'all') {
|
||||
// Remove all duplicates, keeping only first occurrence
|
||||
const seen = new Set<string>();
|
||||
processedLines = lines.filter((line) => {
|
||||
if (seen.has(line)) {
|
||||
return false;
|
||||
}
|
||||
seen.add(line);
|
||||
return true;
|
||||
});
|
||||
} else if (options.mode === 'consecutive') {
|
||||
// Remove only consecutive duplicates
|
||||
processedLines = lines.filter((line, index, arr) => {
|
||||
return index === 0 || line !== arr[index - 1];
|
||||
});
|
||||
} else if (options.mode === 'unique') {
|
||||
// Leave only absolutely unique lines
|
||||
const lineCount = new Map<string, number>();
|
||||
lines.forEach((line) => {
|
||||
lineCount.set(line, (lineCount.get(line) || 0) + 1);
|
||||
});
|
||||
|
||||
processedLines = lines.filter((line) => lineCount.get(line) === 1);
|
||||
}
|
||||
|
||||
// Sort lines if option is selected
|
||||
if (options.sortLines) {
|
||||
processedLines.sort();
|
||||
}
|
||||
|
||||
// Process newlines for output
|
||||
if (options.newlines === 'filter') {
|
||||
// Process newlines as regular lines (already done by default)
|
||||
} else if (options.newlines === 'preserve') {
|
||||
// Make sure empty lines are preserved in the output
|
||||
processedLines = text.split('\n').map((line) => {
|
||||
if (line.trim() === '') return line;
|
||||
return processedLines.includes(line) ? line : '';
|
||||
});
|
||||
}
|
||||
|
||||
return processedLines.join('\n');
|
||||
}
|
||||
|
||||
// Example usage:
|
||||
// const result = removeDuplicateLines(inputText, {
|
||||
// mode: 'all',
|
||||
// newlines: 'filter',
|
||||
// sortLines: false,
|
||||
// trimTextLines: true
|
||||
// });
|
||||
Reference in New Issue
Block a user