fork download 
  1. #include <iostream>
  2. #include <string>
  3. #include <vector>
  4. #include <set>
  5. #include <fstream>
  6. using namespace std;
  7. #define CRC32_POLY 0x82F63B78
  8.  
  9. void crc32( int *, const string &);
  10. void crc32( int *, const char *, unsigned int);
  11. void deldup(const string &, bool output=true);
  12.  
  13.  
  14. int main(int argc, char *argv[]) {
  15. if(argc<2) return 0;
  16. remove("duplicate.txt");
  17. string s=argv[1];
  18. if( s!="/d" && s!="/D" ) {
  19. for(int n=1; n<argc; n++) deldup(argv[n]);
  20. return 0; }
  21. // argv[2]からそれ以降を取り除く
  22. for(int n=3; n<argc; n++) deldup(argv[n], false);
  23. deldup(argv[2]);
  24. return 0; }
  25.  
  26.  
  27. void deldup(const string & path, bool output) {
  28. static vector< set< int> >  M(256);
  29. static ofstream fo;
  30. string buf, str;
  31. char c[5]; int n, x; unsigned char l;
  32. ifstream ifs(path);
  33. while (getline(ifs, str)) {
  34. if(str.empty()) continue;
  35. n = str.length()-1;
  36. if(str[n] =='\r' || str[n] =='\n') {
  37. str.resize(n);
  38. if(str.empty()) continue; }
  39. crc32( (int*)c, str);
  40. c[4] = (char)(str.length() & 0xFF);
  41. l = *(unsigned char*)c;
  42. x = *(int*)&c[1];
  43. if(M[l].find(x) != M[l].end()) {
  44. if(!output) continue; 
  45. if(!fo.is_open()) fo.open("duplicate.txt");
  46. fo<<str+"\n"; 
  47. continue; }
  48. M[l].insert(x);
  49. if(!output) continue;
  50. buf += str + "\n";
  51. if(buf.length()<100*1024) continue;
  52. cout<<buf;
  53. buf=""; }
  54. if(output) cout<<buf; }
  55.  
  56.  
  57.  
  58. void crc32(int *x, const std::string &p) { crc32( x, &p[0], p.size() ); }
  59. void crc32( int *x, const char *p , unsigned int length){
  60. static int init = 0;
  61. static int crcT[256];
  62. if(!init) { 
  63. init=1; 
  64. for (int i=0; i<256; i++) {
  65. int z = i;
  66. for (int j=8; j>0; j--)
  67. if(z&1) z=(z>>1)^CRC32_POLY; else z>>=1;
  68. crcT[i]=z; }}
  69. *x = 0xFFFFFFFF;
  70. for (unsigned int i = 0; i < length; i++) 
  71. *x = ((*x >> 8) & 0x00FFFFFF) ^ crcT[(*x ^ p[i]) & 0xFF ];
  72. *x ^= 0xFFFFFFFF; }
Success #stdin #stdout 0s 15240KB
comments (?)
stdin 
Standard input is empty
stdout 
Standard output is empty
https://ideone.com/ABJNAy
language:
C++14 (gcc 8.3)
created:
5 years ago
visibility:
public

Share or Embed source code

Discover > Sphere Engine API

The brand new service which powers Ideone!

Discover > IDE Widget

Widget for compiling and running the source code in a web browser!