replace flatten with map_while(Result::ok) on iter

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "fqkit"
-version = "0.4.9"
+version = "0.4.10"
 edition = "2021"
 authors = ["sharkLoc <[email protected]>"]
 rust-version = "1.77.2"

README.md

@@ -40,7 +40,7 @@ cargo install --git https://github.com/sharkLoc/fqkit.git
 ```bash
 FqKit -- A simple and cross-platform program for fastq file manipulation
 
-Version: 0.4.9
+Version: 0.4.10
 
 Authors: sharkLoc <[email protected]>
 Source code: https://github.com/sharkLoc/fqkit.git

src/cli/barcode.rs

@@ -15,7 +15,7 @@ fn barcode_list(file: &String, rev_comp: bool) -> Result<HashMap<String, String>
     info!("reading from barcode list file: {}", file);
 
     if rev_comp {
-        for line in fp.lines().map_while(std::io::Result::ok) {
+        for line in fp.lines().map_while(Result::ok) {
             let item = line.split('\t').collect::<Vec<&str>>(); // barcode => sample
             let bar: String = item[0]
                 .chars()
@@ -42,7 +42,7 @@ fn barcode_list(file: &String, rev_comp: bool) -> Result<HashMap<String, String>
             error_flag = "";
         }
     } else {
-        for line in fp.lines().map_while(std::io::Result::ok) {
+        for line in fp.lines().map_while(Result::ok) {
             let item = line.split('\t').collect::<Vec<&str>>();
             let bar: String = item[0]
                 .chars()

src/cli/check.rs

@@ -22,7 +22,7 @@ pub fn check_fastq(
 
     if save {
         let mut out_writer = file_writer(out, compression_level).map(fastq::Writer::new)?;
-        for rec in fp_reader.records().flatten() {
+        for rec in fp_reader.records().map_while(Result::ok) {
             total += 1;
             match rec.check() {
                 Ok(_) => {
@@ -37,7 +37,7 @@ pub fn check_fastq(
         }
         out_writer.flush()?;
     } else {
-        for rec in fp_reader.records().flatten() {
+        for rec in fp_reader.records().map_while(Result::ok) {
             total += 1;
             match rec.check() {
                 Ok(_) => {

src/cli/concat.rs

@@ -46,8 +46,8 @@ pub fn concat_fqstq_lane(
         let fq_reader2 = file_reader(Some(pe.1)).map(fastq::Reader::new)?;
         for (rec1, rec2) in fq_reader1
             .records()
-            .flatten()
-            .zip(fq_reader2.records().flatten())
+            .map_while(Result::ok)
+            .zip(fq_reader2.records().map_while(Result::ok))
         {
             out_writer1.write_record(&rec1)?;
             out_writer2.write_record(&rec2)?;

src/cli/cutadapter.rs

@@ -26,7 +26,7 @@ pub fn cut_adapter(
 
     let mut seqs = HashMap::new();
     let iters = seqfile_reader.records();
-    for rec in iters.flatten() {
+    for rec in iters.map_while(Result::ok) {
         //while let Some(each) = iters.next() {
         //let rec = each?;
         if seqs.contains_key(rec.id()) {

src/cli/filter.rs

@@ -48,7 +48,7 @@ pub fn filter_fastq(
         for (rec1, rec2) in fq_reader1
             .records()
             .flatten()
-            .zip(fq_reader2.records().flatten())
+            .zip(fq_reader2.records().map_while(Result::ok))
         {
             if rec1.seq().iter().filter(|v| v == &&b'N').count() > nbase
                 || rec2.seq().iter().filter(|v| v == &&b'N').count() > nbase
@@ -117,8 +117,8 @@ pub fn filter_fastq(
             let pe_vec: Vec<_> = fq_iter1
                 .by_ref()
                 .take(chunk)
-                .flatten()
-                .zip(fq_iter2.by_ref().take(chunk).flatten())
+                .map_while(Result::ok)
+                .zip(fq_iter2.by_ref().take(chunk).map_while(Result::ok))
                 .collect();
             if pe_vec.is_empty() {
                 break;

src/cli/flatten.rs

@@ -33,7 +33,7 @@ pub fn flatten_fq(
     let fields = get_flag(flag);
     let mut out_writer = file_writer(out, compression_level)?;
 
-    for rec in fq_reader.records().flatten() {
+    for rec in fq_reader.records().map_while(Result::ok) {
         let read = [rec.id().as_bytes(), rec.seq(), "+".as_bytes(), rec.qual()];
         let res = fields.iter().map(|idx| read[*idx]).collect::<Vec<&[u8]>>();
 

src/cli/fq2fa.rs

@@ -23,7 +23,7 @@ pub fn fq2fa(
 
     let mut fo = fasta::Writer::new(file_writer(out, compression_level)?);
     if remove {
-        for rec in fq_reader.records().flatten() {
+        for rec in fq_reader.records().map_while(Result::ok) {
             num += 1;
             fo.write(rec.id(), None, rec.seq())?;
         }

src/cli/fq2sam.rs

@@ -47,7 +47,11 @@ pub fn fastq2sam(
     if let Some(r2) = r2 {
         let fq1 = file_reader(Some(r1)).map(fastq::Reader::new)?;
         let fq2 = file_reader(Some(r2)).map(fastq::Reader::new)?;
-        for (rec1, rec2) in fq1.records().flatten().zip(fq2.records().flatten()) {
+        for (rec1, rec2) in fq1
+            .records()
+            .map_while(Result::ok)
+            .zip(fq2.records().map_while(Result::ok))
+        {
             sam.write_fmt(format_args!(
                 "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\tRG:Z:{}\n",
                 rec1.id(),
@@ -81,7 +85,7 @@ pub fn fastq2sam(
         }
     } else {
         let fq = file_reader(Some(r1)).map(fastq::Reader::new)?;
-        for rec in fq.records().flatten() {
+        for rec in fq.records().map_while(Result::ok) {
             sam.write_fmt(format_args!(
                 "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\tRG:Z:{}\n",
                 rec.id(),

src/cli/fqscore.rs

@@ -37,7 +37,7 @@ pub fn phred_score(
     }
 
     let mut fq_writer = file_writer(out, compression_level).map(fastq::Writer::new)?;
-    for rec in fq_reader.records().flatten() {
+    for rec in fq_reader.records().map_while(Result::ok) {
         let mut qual = vec![];
         if to33 {
             for q in rec.qual() {

src/cli/gcplot.rs

@@ -32,7 +32,7 @@ pub fn gc_content(
     let mut fo = file_writer(output, compression_level)?;
     let mut df_hash = HashMap::new();
 
-    for rec in fq_reader.records().flatten() {
+    for rec in fq_reader.records().map_while(Result::ok) {
         let gc_count = rec
             .seq()
             .iter()

src/cli/grep.rs

@@ -26,7 +26,7 @@ pub fn grep_fastq(
     let mut ids = vec![];
     let fp_id = file_reader(Some(list))?;
     info!("reading reads id from file: {}", list);
-    for id in fp_id.lines().map_while(std::io::Result::ok) {
+    for id in fp_id.lines().map_while(Result::ok) {
         ids.push(id);
     }
     if ids.is_empty() {
@@ -40,7 +40,7 @@ pub fn grep_fastq(
         info!("reads write to stdout");
     }
     let mut fo = file_writer(out, compression_level).map(fastq::Writer::new)?;
-    for rec in fq_reader.records().flatten() {
+    for rec in fq_reader.records().map_while(Result::ok) {
         let name = if full_name {
             if let Some(desc) = rec.desc() {
                 format!("{} {}", rec.id(), desc)

src/cli/kmer.rs

@@ -23,11 +23,11 @@ pub fn kmer_count(
     let mut writer = file_writer(output, compression_level)?;
     let mut kmers = HashMap::new();
 
-    for rec in reader.records().flatten() {
+    for rec in reader.records().map_while(Result::ok) {
         let (mut sidx, mut eidx) = (0, kmer_len);
         let khash = nthash(rec.seq(), kmer_len);
         let len = rec.seq().len();
-        
+
         while eidx <= len {
             let kseq = &rec.seq()[sidx..eidx];
             let khash_this = nthash(kseq, kmer_len)[0];

src/cli/length.rs

@@ -23,7 +23,7 @@ pub fn fq_length(
     }
 
     let mut fo = file_writer(out, compression_level)?;
-    for rec in fp_reader.records().flatten() {
+    for rec in fp_reader.records().map_while(Result::ok) {
         let rlen = rec.seq().len();
         *reads_len.entry(rlen).or_insert(0usize) += 1;
         total += 1;

src/cli/mask.rs

@@ -25,7 +25,7 @@ pub fn mask_fastq(
     info!("mask low quality bases with: {}", nt);
 
     let mut fp_writer = file_writer(out, compression_level).map(fastq::Writer::new)?;
-    for rec in fp_reader.records().flatten() {
+    for rec in fp_reader.records().map_while(Result::ok) {
         let score_min = rec.qual().iter().min().unwrap() - phred;
         if score_min > qual_limit {
             fp_writer.write_record(&rec)?;

src/cli/merge.rs

@@ -21,8 +21,8 @@ pub fn interleaved(
     let mut fq_writer = fastq::Writer::new(file_writer(out, compression_level)?);
     for (rec1, rec2) in fq1_reader
         .records()
-        .flatten()
-        .zip(fq2_reader.records().flatten())
+        .map_while(Result::ok)
+        .zip(fq2_reader.records().map_while(Result::ok))
     {
         num += 1;
         fq_writer.write(rec1.id(), rec1.desc(), rec1.seq(), rec1.qual())?;

src/cli/plot.rs

@@ -16,7 +16,7 @@ pub fn cycle_data(file: Option<&String>) -> Result<Vec<BTreeMap<usize, f64>>> {
         cyc.push(BTreeMap::new());
     }
 
-    for (idx, line) in fp.lines().map_while(std::io::Result::ok).enumerate() {
+    for (idx, line) in fp.lines().map_while(Result::ok).enumerate() {
         if idx == 0 {
             continue;
         }

src/cli/range.rs

@@ -23,7 +23,12 @@ pub fn range_fastq(
     info!("get {} records", take);
 
     let mut fp_writer = file_writer(output, compression_level).map(fastq::Writer::new)?;
-    for rec in fp_reader.records().skip(skip).take(take).flatten() {
+    for rec in fp_reader
+        .records()
+        .skip(skip)
+        .take(take)
+        .map_while(Result::ok)
+    {
         fp_writer.write_record(&rec)?;
     }
     fp_writer.flush()?;

src/cli/remove.rs

@@ -39,15 +39,15 @@ pub fn remove_read(
 
     let mut fq_writer = fastq::Writer::new(file_writer(out, compression_level)?);
     if rm {
-        for rec in fq_reader.records().flatten() {
+        for rec in fq_reader.records().map_while(Result::ok) {
             if !ids.contains(&rec.id().to_string()) {
                 fq_writer.write(rec.id(), rec.desc(), rec.seq(), rec.qual())?;
             }
         }
         fq_writer.flush()?;
     } else {
         let mut rm_writer = fastq::Writer::new(file_writer(Some(save), compression_level)?);
-        for rec in fq_reader.records().flatten() {
+        for rec in fq_reader.records().map_while(Result::ok) {
             if !ids.contains(&rec.id().to_string()) {
                 fq_writer.write(rec.id(), rec.desc(), rec.seq(), rec.qual())?;
             } else {

src/cli/rename.rs

@@ -26,7 +26,7 @@ pub fn rename_fastq(
     let mut n: usize = 0;
 
     if let Some(pre) = prefix {
-        for rec in fp.records().flatten() {
+        for rec in fp.records().map_while(Result::ok) {
             n += 1;
             /*let newid = match label {
                 Some(x) => {
@@ -53,7 +53,7 @@ pub fn rename_fastq(
         }
         fo.flush()?;
     } else {
-        for rec in fp.records().flatten() {
+        for rec in fp.records().map_while(Result::ok) {
             n += 1;
             /*let newid = if let Some(x) = label {
                 if before {

src/cli/reverse.rs

@@ -29,7 +29,7 @@ pub fn reverse_comp_seq(
     ]);
     let mut out_writer = file_writer(out, compression_level).map(fastq::Writer::new)?;
 
-    for rec in fq_reader.records().flatten() {
+    for rec in fq_reader.records().map_while(Result::ok) {
         let rev_seq = rec.seq().iter().copied().rev().collect::<Vec<u8>>();
         let rev_qual = rec.qual().iter().copied().rev().collect::<Vec<u8>>();
 

src/cli/search.rs

@@ -54,7 +54,7 @@ pub fn search_fq(
             .case_insensitive(case)
             .build()
             .unwrap();
-        for rec in fq_reader.records().flatten() {
+        for rec in fq_reader.records().map_while(Result::ok) {
             let fq_str = std::str::from_utf8(rec.seq()).unwrap();
             if invert_match {
                 if !re.is_match(fq_str) {
@@ -71,7 +71,7 @@ pub fn search_fq(
         let (tx, rx) = unbounded();
         let mut fqiter = fq_reader.records();
         loop {
-            let chunks: Vec<_> = fqiter.by_ref().take(chunk).flatten().collect();
+            let chunks: Vec<_> = fqiter.by_ref().take(chunk).map_while(Result::ok).collect();
             if chunks.is_empty() {
                 break;
             }

src/cli/select.rs

@@ -20,11 +20,11 @@ pub fn select_pe_fastq(
     info!("read forward reads from file: {}", fq1);
     info!("read reverse reads from file: {}", fq2);
 
-    for rec in fq_reader1.records().flatten() {
+    for rec in fq_reader1.records().map_while(Result::ok) {
         let k = rec.id().to_owned();
         read1_id.entry(k).or_insert(());
     }
-    for rec in fq_reader2.records().flatten() {
+    for rec in fq_reader2.records().map_while(Result::ok) {
         let k = rec.id().to_owned();
         read2_id.entry(k).or_insert(());
     }
@@ -35,7 +35,7 @@ pub fn select_pe_fastq(
     let (mut pe_r1, mut pe_r2) = (0usize, 0usize);
 
     let fq_reader1 = file_reader(Some(fq1)).map(fastq::Reader::new)?;
-    for rec in fq_reader1.records().flatten() {
+    for rec in fq_reader1.records().map_while(Result::ok) {
         if read1_id.contains_key(rec.id()) && read2_id.contains_key(rec.id()) {
             pe_r1 += 1;
             out_writer1.write_record(&rec)?;
@@ -44,7 +44,7 @@ pub fn select_pe_fastq(
     out_writer1.flush()?;
 
     let fq_reader2 = file_reader(Some(fq2)).map(fastq::Reader::new)?;
-    for rec in fq_reader2.records().flatten() {
+    for rec in fq_reader2.records().map_while(Result::ok) {
         if read2_id.contains_key(rec.id()) && read1_id.contains_key(rec.id()) {
             pe_r2 += 1;
             out_writer2.write_record(&rec)?;

src/cli/shuffle.rs

@@ -24,7 +24,7 @@ pub fn shuffle_fastq(
     info!("rand seed: {}", seed);
 
     let mut vec_reads = vec![];
-    for rec in fq_reader.records().flatten() {
+    for rec in fq_reader.records().map_while(Result::ok) {
         vec_reads.push(rec);
     }
 

src/cli/size.rs

@@ -65,7 +65,7 @@ pub fn size_fastq(
     let mut bases = 0usize;
 
     if ncpu == 0 || ncpu == 1 {
-        for rec in fq_reader.records().flatten() {
+        for rec in fq_reader.records().map_while(Result::ok) {
             base.read += 1;
             for nt in rec.seq().iter() {
                 match *nt {
@@ -83,7 +83,7 @@ pub fn size_fastq(
         let (tx, rx) = unbounded();
         let mut fqiter = fq_reader.records();
         loop {
-            let chunk: Vec<_> = fqiter.by_ref().take(chunk).flatten().collect();
+            let chunk: Vec<_> = fqiter.by_ref().take(chunk).map_while(Result::ok).collect();
             if chunk.is_empty() {
                 break;
             }

src/cli/slide.rs

@@ -24,7 +24,7 @@ pub fn slide_fastq(
     info!("step size: {}", step);
     let mut fq_writer = file_writer(out, compression_level).map(fastq::Writer::new)?;
     let mut window = wind;
-    for rec in fq_reader.records().flatten() {
+    for rec in fq_reader.records().map_while(Result::ok) {
         let seq = rec.seq();
         let qual = rec.qual();
         let len = seq.len();

src/cli/sort.rs

@@ -50,7 +50,7 @@ pub fn sort_fastq(
     }
 
     let mut vec_reads = vec![];
-    for rec in fq_reader.records().flatten() {
+    for rec in fq_reader.records().map_while(Result::ok) {
         vec_reads.push(rec);
     }
     info!("all records has been readed into memory, start sort ...");

src/cli/split.rs

@@ -63,7 +63,7 @@ pub fn split_interleaved(
 
     let mut num = 0usize;
     let mut flag = true;
-    for rec in fq_reader.records().flatten() {
+    for rec in fq_reader.records().map_while(Result::ok) {
         num += 1;
         if flag {
             fh1.write(rec.id(), rec.desc(), rec.seq(), rec.qual())?;