Project Full coverage report
Current view: Cores-VeeR-EL2—Cores-VeeR-EL2—design—lsu—el2_lsu_lsc_ctl.sv Coverage Hit Total
Test Date: 19-09-2024 Toggle 78.8% 67 85
Test: all Branch 100.0% 17 17

            Line data    Source code
       1              : // SPDX-License-Identifier: Apache-2.0
       2              : // Copyright 2020 Western Digital Corporation or its affiliates.
       3              : //
       4              : // Licensed under the Apache License, Version 2.0 (the "License");
       5              : // you may not use this file except in compliance with the License.
       6              : // You may obtain a copy of the License at
       7              : //
       8              : // http://www.apache.org/licenses/LICENSE-2.0
       9              : //
      10              : // Unless required by applicable law or agreed to in writing, software
      11              : // distributed under the License is distributed on an "AS IS" BASIS,
      12              : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      13              : // See the License for the specific language governing permissions and
      14              : // limitations under the License.
      15              : 
      16              : //********************************************************************************
      17              : // $Id$
      18              : //
      19              : //
      20              : // Owner:
      21              : // Function: LSU control
      22              : // Comments:
      23              : //
      24              : //
      25              : // DC1 -> DC2 -> DC3 -> DC4 (Commit)
      26              : //
      27              : //********************************************************************************
      28              : module el2_lsu_lsc_ctl
      29              : import el2_pkg::*;
      30              : #(
      31              : `include "el2_param.vh"
      32              :  )(
      33          316 :    input logic                rst_l,                     // reset, active low
      34            0 :    input logic                clk_override,              // Override non-functional clock gating
      35     61843746 :    input logic                clk,                       // Clock only while core active.  Through one clock header.  For flops with    second clock header built in.  Connected to ACTIVE_L2CLK.
      36              : 
      37              :    // clocks per pipe
      38     61843746 :    input logic                lsu_c1_m_clk,
      39     61843746 :    input logic                lsu_c1_r_clk,
      40     61843746 :    input logic                lsu_c2_m_clk,
      41     61843746 :    input logic                lsu_c2_r_clk,
      42     61843746 :    input logic                lsu_store_c1_m_clk,
      43              : 
      44            0 :    input logic [31:0]         lsu_ld_data_r,             // Load data R-stage
      45        24694 :    input logic [31:0]         lsu_ld_data_corr_r,        // ECC corrected data R-stage
      46            4 :    input logic                lsu_single_ecc_error_r,    // ECC single bit error R-stage
      47            4 :    input logic                lsu_double_ecc_error_r,    // ECC double bit error R-stage
      48              : 
      49        54452 :    input logic [31:0]         lsu_ld_data_m,             // Load data M-stage
      50            4 :    input logic                lsu_single_ecc_error_m,    // ECC single bit error M-stage
      51            4 :    input logic                lsu_double_ecc_error_m,    // ECC double bit error M-stage
      52              : 
      53        58638 :    input logic                flush_m_up,                // Flush M and D stage
      54        29654 :    input logic                flush_r,                   // Flush R-stage
      55        36568 :    input logic                ldst_dual_d,               // load/store is unaligned at 32 bit boundary D-stage
      56        36568 :    input logic                ldst_dual_m,               // load/store is unaligned at 32 bit boundary M-stage
      57        36568 :    input logic                ldst_dual_r,               // load/store is unaligned at 32 bit boundary R-stage
      58              : 
      59       413833 :    input logic [31:0]         exu_lsu_rs1_d,             // address
      60        81386 :    input logic [31:0]         exu_lsu_rs2_d,             // store data
      61              : 
      62       623945 :    input el2_lsu_pkt_t       lsu_p,                     // lsu control packet
      63      2276073 :    input logic                dec_lsu_valid_raw_d,       // Raw valid for address computation
      64       270518 :    input logic [11:0]         dec_lsu_offset_d,          // 12b offset for load/store addresses
      65              : 
      66            0 :    input  logic [31:0]        picm_mask_data_m,          // PIC data M-stage
      67          200 :    input  logic [31:0]        bus_read_data_m,           // the bus return data
      68        38395 :    output logic [31:0]        lsu_result_m,              // lsu load data
      69        29134 :    output logic [31:0]        lsu_result_corr_r,         // This is the ECC corrected data going to RF
      70              :    // lsu address down the pipe
      71       594111 :    output logic [31:0]        lsu_addr_d,
      72       594110 :    output logic [31:0]        lsu_addr_m,
      73       594109 :    output logic [31:0]        lsu_addr_r,
      74              :    // lsu address down the pipe - needed to check unaligned
      75       594149 :    output logic [31:0]        end_addr_d,
      76       594510 :    output logic [31:0]        end_addr_m,
      77       594509 :    output logic [31:0]        end_addr_r,
      78              :    // store data down the pipe
      79        81386 :    output logic [31:0]        store_data_m,
      80              : 
      81            0 :    input  logic [31:0]         dec_tlu_mrac_ff,          // CSR for memory region control
      82           60 :    output logic                lsu_exc_m,                // Access or misaligned fault
      83        29240 :    output logic                is_sideeffects_m,         // is sideffects space
      84      2279496 :    output logic                lsu_commit_r,             // lsu instruction in r commits
      85            4 :    output logic                lsu_single_ecc_error_incr,// LSU inc SB error counter
      86            4 :    output el2_lsu_error_pkt_t lsu_error_pkt_r,          // lsu exception packet
      87              : 
      88        24694 :    output logic [31:1]         lsu_fir_addr,             // fast interrupt address
      89            0 :    output logic [1:0]          lsu_fir_error,            // Error during fast interrupt lookup
      90              : 
      91              :    // address in dccm/pic/external per pipe stage
      92       614420 :    output logic               addr_in_dccm_d,
      93       614420 :    output logic               addr_in_dccm_m,
      94       614420 :    output logic               addr_in_dccm_r,
      95              : 
      96            0 :    output logic               addr_in_pic_d,
      97            0 :    output logic               addr_in_pic_m,
      98            0 :    output logic               addr_in_pic_r,
      99              : 
     100       614746 :    output logic               addr_external_m,
     101              : 
     102              :    // DMA slave
     103            0 :    input logic                dma_dccm_req,
     104            0 :    input logic [31:0]         dma_mem_addr,
     105            0 :    input logic [2:0]          dma_mem_sz,
     106           18 :    input logic                dma_mem_write,
     107           12 :    input logic [63:0]         dma_mem_wdata,
     108              : 
     109              :    // Store buffer related signals
     110       478232 :    output el2_lsu_pkt_t      lsu_pkt_d,
     111       478184 :    output el2_lsu_pkt_t      lsu_pkt_m,
     112       478181 :    output el2_lsu_pkt_t      lsu_pkt_r,
     113              : 
     114       167732 :     input logic lsu_pmp_error_start,
     115       167732 :     input logic lsu_pmp_error_end,
     116              : 
     117            0 :    input  logic               scan_mode                  // Scan mode
     118              : 
     119              :    );
     120              : 
     121           13 :    logic [31:3]        end_addr_pre_m, end_addr_pre_r;
     122       594111 :    logic [31:0]        full_addr_d;
     123       594149 :    logic [31:0]        full_end_addr_d;
     124       476238 :    logic [31:0]        lsu_rs1_d;
     125       270136 :    logic [11:0]        lsu_offset_d;
     126       476238 :    logic [31:0]        rs1_d;
     127       270136 :    logic [11:0]        offset_d;
     128       283348 :    logic [12:0]        end_addr_offset_d;
     129            0 :    logic [2:0]         addr_offset_d;
     130              : 
     131           12 :    logic [63:0]        dma_mem_wdata_shifted;
     132       614747 :    logic               addr_external_d;
     133       614746 :    logic               addr_external_r;
     134            0 :    logic               access_fault_d, misaligned_fault_d;
     135            0 :    logic               access_fault_m, misaligned_fault_m;
     136              : 
     137            0 :    logic               fir_dccm_access_error_d, fir_nondccm_access_error_d;
     138            0 :    logic               fir_dccm_access_error_m, fir_nondccm_access_error_m;
     139              : 
     140            0 :    logic [3:0]         exc_mscause_d, exc_mscause_m;
     141       476238 :    logic [31:0]        rs1_d_raw;
     142        81386 :    logic [31:0]        store_data_d, store_data_pre_m, store_data_m_in;
     143          198 :    logic [31:0]        bus_read_data_r;
     144              : 
     145           18 :    el2_lsu_pkt_t           dma_pkt_d;
     146       478184 :    el2_lsu_pkt_t           lsu_pkt_m_in, lsu_pkt_r_in;
     147            4 :    el2_lsu_error_pkt_t     lsu_error_pkt_m;
     148              : 
     149              : 
     150              :    // Premux the rs1/offset for dma
     151              :    assign lsu_rs1_d[31:0]    = dec_lsu_valid_raw_d ? exu_lsu_rs1_d[31:0] : dma_mem_addr[31:0];
     152              :    assign lsu_offset_d[11:0] = dec_lsu_offset_d[11:0] & {12{dec_lsu_valid_raw_d}};
     153              :    assign rs1_d_raw[31:0]    = lsu_rs1_d[31:0];
     154              :    assign offset_d[11:0]     = lsu_offset_d[11:0];
     155              : 
     156              :    assign rs1_d[31:0] = (lsu_pkt_d.load_ldst_bypass_d) ? lsu_result_m[31:0] : rs1_d_raw[31:0];
     157              : 
     158              :    // generate the ls address
     159              :    rvlsadder   lsadder  (.rs1(rs1_d[31:0]),
     160              :                        .offset(offset_d[11:0]),
     161              :                        .dout(full_addr_d[31:0])
     162              :                        );
     163              : 
     164              :    // Module to generate the memory map of the address
     165              :    el2_lsu_addrcheck addrcheck (
     166              :               .start_addr_d(full_addr_d[31:0]),
     167              :               .end_addr_d(full_end_addr_d[31:0]),
     168              :               .rs1_region_d(rs1_d[31:28]),
     169              :               .*
     170              :   );
     171              : 
     172              :    // Calculate start/end address for load/store
     173              :    assign addr_offset_d[2:0]      = ({3{lsu_pkt_d.half}} & 3'b01) | ({3{lsu_pkt_d.word}} & 3'b11) | ({3{lsu_pkt_d.dword}} & 3'b111);
     174              :    assign end_addr_offset_d[12:0] = {offset_d[11],offset_d[11:0]} + {9'b0,addr_offset_d[2:0]};
     175              :    assign full_end_addr_d[31:0]   = rs1_d[31:0] + {{19{end_addr_offset_d[12]}},end_addr_offset_d[12:0]};
     176              :    assign end_addr_d[31:0]        = full_end_addr_d[31:0];
     177              :    assign lsu_exc_m               = access_fault_m | misaligned_fault_m;
     178              : 
     179              :    // Goes to TLU to increment the ECC error counter
     180              :    assign lsu_single_ecc_error_incr = (lsu_single_ecc_error_r & ~lsu_double_ecc_error_r) & (lsu_commit_r | lsu_pkt_r.dma) & lsu_pkt_r.valid;
     181              : 
     182              :    if (pt.LOAD_TO_USE_PLUS1 == 1) begin: L2U_Plus1_1
     183              :       logic               access_fault_r, misaligned_fault_r;
     184              :       logic [3:0]         exc_mscause_r;
     185              :       logic               fir_dccm_access_error_r, fir_nondccm_access_error_r;
     186              : 
     187              :       // Generate exception packet
     188              :       assign lsu_error_pkt_r.exc_valid = (access_fault_r | misaligned_fault_r | lsu_double_ecc_error_r) & lsu_pkt_r.valid & ~lsu_pkt_r.dma & ~lsu_pkt_r.fast_int;
     189              :       assign lsu_error_pkt_r.single_ecc_error = lsu_single_ecc_error_r & ~lsu_error_pkt_r.exc_valid & ~lsu_pkt_r.dma;
     190              :       assign lsu_error_pkt_r.inst_type = lsu_pkt_r.store;
     191              :       assign lsu_error_pkt_r.exc_type  = ~misaligned_fault_r;
     192              :       assign lsu_error_pkt_r.mscause[3:0] = (lsu_double_ecc_error_r & ~misaligned_fault_r & ~access_fault_r) ? 4'h1 : exc_mscause_r[3:0];
     193              :       assign lsu_error_pkt_r.addr[31:0] = lsu_addr_r[31:0];
     194              : 
     195              :       assign lsu_fir_error[1:0] = fir_nondccm_access_error_r ? 2'b11 : (fir_dccm_access_error_r ? 2'b10 : ((lsu_pkt_r.fast_int & lsu_double_ecc_error_r) ? 2'b01 : 2'b00));
     196              : 
     197              :       rvdff #(1) access_fault_rff             (.din(access_fault_m),             .dout(access_fault_r),             .clk(lsu_c1_r_clk), .*);
     198              :       rvdff #(1) misaligned_fault_rff         (.din(misaligned_fault_m),         .dout(misaligned_fault_r),         .clk(lsu_c1_r_clk), .*);
     199              :       rvdff #(4) exc_mscause_rff              (.din(exc_mscause_m[3:0]),         .dout(exc_mscause_r[3:0]),         .clk(lsu_c1_r_clk), .*);
     200              :       rvdff #(1) fir_dccm_access_error_mff    (.din(fir_dccm_access_error_m),    .dout(fir_dccm_access_error_r),    .clk(lsu_c1_r_clk), .*);
     201              :       rvdff #(1) fir_nondccm_access_error_mff (.din(fir_nondccm_access_error_m), .dout(fir_nondccm_access_error_r), .clk(lsu_c1_r_clk), .*);
     202              : 
     203              :    end else begin: L2U_Plus1_0
     204              :       logic [1:0] lsu_fir_error_m;
     205              : 
     206              :       // Generate exception packet
     207              :       assign lsu_error_pkt_m.exc_valid = (access_fault_m | misaligned_fault_m | lsu_double_ecc_error_m) & lsu_pkt_m.valid & ~lsu_pkt_m.dma & ~lsu_pkt_m.fast_int & ~flush_m_up;
     208              :       assign lsu_error_pkt_m.single_ecc_error = lsu_single_ecc_error_m & ~lsu_error_pkt_m.exc_valid & ~lsu_pkt_m.dma;
     209              :       assign lsu_error_pkt_m.inst_type = lsu_pkt_m.store;
     210              :       assign lsu_error_pkt_m.exc_type  = ~misaligned_fault_m;
     211              :       assign lsu_error_pkt_m.mscause[3:0] = (lsu_double_ecc_error_m & ~misaligned_fault_m & ~access_fault_m) ? 4'h1 : exc_mscause_m[3:0];
     212              :       assign lsu_error_pkt_m.addr[31:0] = lsu_addr_m[31:0];
     213              : 
     214              :       assign lsu_fir_error_m[1:0] = fir_nondccm_access_error_m ? 2'b11 : (fir_dccm_access_error_m ? 2'b10 : ((lsu_pkt_m.fast_int & lsu_double_ecc_error_m) ? 2'b01 : 2'b00));
     215              : 
     216              :       rvdff  #(1)                             lsu_exc_valid_rff       (.*, .din(lsu_error_pkt_m.exc_valid),                        .dout(lsu_error_pkt_r.exc_valid),                        .clk(lsu_c2_r_clk));
     217              :       rvdff  #(1)                             lsu_single_ecc_error_rff(.*, .din(lsu_error_pkt_m.single_ecc_error),                 .dout(lsu_error_pkt_r.single_ecc_error),                 .clk(lsu_c2_r_clk));
     218              :       rvdffe #($bits(el2_lsu_error_pkt_t)-2) lsu_error_pkt_rff       (.*, .din(lsu_error_pkt_m[$bits(el2_lsu_error_pkt_t)-1:2]), .dout(lsu_error_pkt_r[$bits(el2_lsu_error_pkt_t)-1:2]), .en(lsu_error_pkt_m.exc_valid | lsu_error_pkt_m.single_ecc_error | clk_override));
     219              :       rvdff #(2)                              lsu_fir_error_rff       (.*, .din(lsu_fir_error_m[1:0]),                             .dout(lsu_fir_error[1:0]),                               .clk(lsu_c2_r_clk));
     220              :    end
     221              : 
     222              :    //Create DMA packet
     223          317 :    always_comb begin
     224          317 :       dma_pkt_d = '0;
     225          317 :       dma_pkt_d.valid   = dma_dccm_req;
     226          317 :       dma_pkt_d.dma     = 1'b1;
     227          317 :       dma_pkt_d.store   = dma_mem_write;
     228          317 :       dma_pkt_d.load    = ~dma_mem_write;
     229          317 :       dma_pkt_d.by      = (dma_mem_sz[2:0] == 3'b0);
     230          317 :       dma_pkt_d.half    = (dma_mem_sz[2:0] == 3'b1);
     231          317 :       dma_pkt_d.word    = (dma_mem_sz[2:0] == 3'b10);
     232          317 :       dma_pkt_d.dword   = (dma_mem_sz[2:0] == 3'b11);
     233              :    end
     234              : 
     235          317 :    always_comb begin
     236          317 :       lsu_pkt_d = dec_lsu_valid_raw_d ? lsu_p : dma_pkt_d;
     237          317 :       lsu_pkt_m_in = lsu_pkt_d;
     238          317 :       lsu_pkt_r_in = lsu_pkt_m;
     239              : 
     240          317 :       lsu_pkt_d.valid = (lsu_p.valid & ~(flush_m_up & ~lsu_p.fast_int)) | dma_dccm_req;
     241          317 :       lsu_pkt_m_in.valid = lsu_pkt_d.valid & ~(flush_m_up & ~lsu_pkt_d.dma);
     242          317 :       lsu_pkt_r_in.valid = lsu_pkt_m.valid & ~(flush_m_up & ~lsu_pkt_m.dma) ;
     243              :    end
     244              : 
     245              :    // C2 clock for valid and C1 for other bits of packet
     246              :    rvdff #(1) lsu_pkt_vldmff (.*, .din(lsu_pkt_m_in.valid), .dout(lsu_pkt_m.valid), .clk(lsu_c2_m_clk));
     247              :    rvdff #(1) lsu_pkt_vldrff (.*, .din(lsu_pkt_r_in.valid), .dout(lsu_pkt_r.valid), .clk(lsu_c2_r_clk));
     248              : 
     249              :    rvdff #($bits(el2_lsu_pkt_t)-1) lsu_pkt_mff (.*, .din(lsu_pkt_m_in[$bits(el2_lsu_pkt_t)-1:1]), .dout(lsu_pkt_m[$bits(el2_lsu_pkt_t)-1:1]), .clk(lsu_c1_m_clk));
     250              :    rvdff #($bits(el2_lsu_pkt_t)-1) lsu_pkt_rff (.*, .din(lsu_pkt_r_in[$bits(el2_lsu_pkt_t)-1:1]), .dout(lsu_pkt_r[$bits(el2_lsu_pkt_t)-1:1]), .clk(lsu_c1_r_clk));
     251              : 
     252              : 
     253              : 
     254              :    if (pt.LOAD_TO_USE_PLUS1 == 1) begin: L2U1_Plus1_1
     255              :       logic [31:0] lsu_ld_datafn_r, lsu_ld_datafn_corr_r;
     256              : 
     257              :       assign lsu_ld_datafn_r[31:0]  = addr_external_r ? bus_read_data_r[31:0] : lsu_ld_data_r[31:0];
     258              :       assign lsu_ld_datafn_corr_r[31:0]  = addr_external_r ? bus_read_data_r[31:0] : lsu_ld_data_corr_r[31:0];
     259              : 
     260              :       // this is really R stage signal
     261              :       assign lsu_result_m[31:0] = ({32{ lsu_pkt_r.unsign & lsu_pkt_r.by  }} & {24'b0,lsu_ld_datafn_r[7:0]}) |
     262              :                                                                     ({32{ lsu_pkt_r.unsign & lsu_pkt_r.half}} & {16'b0,lsu_ld_datafn_r[15:0]}) |
     263              :                                                                     ({32{~lsu_pkt_r.unsign & lsu_pkt_r.by  }} & {{24{  lsu_ld_datafn_r[7]}}, lsu_ld_datafn_r[7:0]}) |
     264              :                                                                     ({32{~lsu_pkt_r.unsign & lsu_pkt_r.half}} & {{16{  lsu_ld_datafn_r[15]}},lsu_ld_datafn_r[15:0]}) |
     265              :                                                                     ({32{lsu_pkt_r.word}}                     & lsu_ld_datafn_r[31:0]);
     266              : 
     267              :       // this signal is used for gpr update
     268              :       assign lsu_result_corr_r[31:0] = ({32{ lsu_pkt_r.unsign & lsu_pkt_r.by  }} & {24'b0,lsu_ld_datafn_corr_r[7:0]}) |
     269              :                                                                               ({32{ lsu_pkt_r.unsign & lsu_pkt_r.half}} & {16'b0,lsu_ld_datafn_corr_r[15:0]}) |
     270              :                                                                               ({32{~lsu_pkt_r.unsign & lsu_pkt_r.by  }} & {{24{  lsu_ld_datafn_corr_r[7]}}, lsu_ld_datafn_corr_r[7:0]}) |
     271              :                                                                               ({32{~lsu_pkt_r.unsign & lsu_pkt_r.half}} & {{16{  lsu_ld_datafn_corr_r[15]}},lsu_ld_datafn_corr_r[15:0]}) |
     272              :                                                                               ({32{lsu_pkt_r.word}}                     & lsu_ld_datafn_corr_r[31:0]);
     273              : 
     274              :    end else begin: L2U1_Plus1_0 // block: L2U1_Plus1_1
     275              :       logic [31:0] lsu_ld_datafn_m, lsu_ld_datafn_corr_r;
     276              : 
     277              :       assign lsu_ld_datafn_m[31:0] = addr_external_m ? bus_read_data_m[31:0] : lsu_ld_data_m[31:0];
     278              :       assign lsu_ld_datafn_corr_r[31:0] = addr_external_r ? bus_read_data_r[31:0] : lsu_ld_data_corr_r[31:0];
     279              : 
     280              :       // this result must look at prior stores and merge them in
     281              :       assign lsu_result_m[31:0] = ({32{ lsu_pkt_m.unsign & lsu_pkt_m.by  }} & {24'b0,lsu_ld_datafn_m[7:0]}) |
     282              :                                                                     ({32{ lsu_pkt_m.unsign & lsu_pkt_m.half}} & {16'b0,lsu_ld_datafn_m[15:0]}) |
     283              :                                                                     ({32{~lsu_pkt_m.unsign & lsu_pkt_m.by  }} & {{24{  lsu_ld_datafn_m[7]}}, lsu_ld_datafn_m[7:0]}) |
     284              :                                                                     ({32{~lsu_pkt_m.unsign & lsu_pkt_m.half}} & {{16{  lsu_ld_datafn_m[15]}},lsu_ld_datafn_m[15:0]}) |
     285              :                                                                     ({32{lsu_pkt_m.word}}                     & lsu_ld_datafn_m[31:0]);
     286              : 
     287              :       // this signal is used for gpr update
     288              :       assign lsu_result_corr_r[31:0] = ({32{ lsu_pkt_r.unsign & lsu_pkt_r.by  }} & {24'b0,lsu_ld_datafn_corr_r[7:0]}) |
     289              :                                                                               ({32{ lsu_pkt_r.unsign & lsu_pkt_r.half}} & {16'b0,lsu_ld_datafn_corr_r[15:0]}) |
     290              :                                                                               ({32{~lsu_pkt_r.unsign & lsu_pkt_r.by  }} & {{24{  lsu_ld_datafn_corr_r[7]}}, lsu_ld_datafn_corr_r[7:0]}) |
     291              :                                                                               ({32{~lsu_pkt_r.unsign & lsu_pkt_r.half}} & {{16{  lsu_ld_datafn_corr_r[15]}},lsu_ld_datafn_corr_r[15:0]}) |
     292              :                                                                               ({32{lsu_pkt_r.word}}                     & lsu_ld_datafn_corr_r[31:0]);
     293              :    end
     294              : 
     295              :    // Fast interrupt address
     296              :    assign lsu_fir_addr[31:1]    = lsu_ld_data_corr_r[31:1];
     297              : 
     298              :    // absence load/store all 0's
     299              :    assign lsu_addr_d[31:0] = full_addr_d[31:0];
     300              : 
     301              :    // Interrupt as a flush source allows the WB to occur
     302              :    assign lsu_commit_r = lsu_pkt_r.valid & (lsu_pkt_r.store | lsu_pkt_r.load) & ~flush_r & ~lsu_pkt_r.dma;
     303              : 
     304              :    assign dma_mem_wdata_shifted[63:0] = 64'(dma_mem_wdata[63:0] >> {dma_mem_addr[2:0], 3'b000});   // Shift the dma data to lower bits to make it consistent to lsu stores
     305              :    assign store_data_d[31:0] = dma_dccm_req ? dma_mem_wdata_shifted[31:0] : exu_lsu_rs2_d[31:0];  // Write to PIC still happens in r stage
     306              : 
     307              :    assign store_data_m_in[31:0] = (lsu_pkt_d.store_data_bypass_d) ? lsu_result_m[31:0] : store_data_d[31:0];
     308              : 
     309              :    assign store_data_m[31:0] = (picm_mask_data_m[31:0] | {32{~addr_in_pic_m}}) & ((lsu_pkt_m.store_data_bypass_m) ? lsu_result_m[31:0] : store_data_pre_m[31:0]);
     310              : 
     311              : 
     312              :    rvdff #(32)  sdmff (.*, .din(store_data_m_in[31:0]), .dout(store_data_pre_m[31:0]),                       .clk(lsu_store_c1_m_clk));
     313              : 
     314              :    rvdff #(32) samff (.*, .din(lsu_addr_d[31:0]), .dout(lsu_addr_m[31:0]), .clk(lsu_c1_m_clk));
     315              :    rvdff #(32) sarff (.*, .din(lsu_addr_m[31:0]), .dout(lsu_addr_r[31:0]), .clk(lsu_c1_r_clk));
     316              : 
     317              :    assign end_addr_m[31:3] = ldst_dual_m ? end_addr_pre_m[31:3] : lsu_addr_m[31:3];       // This is for power saving
     318              :    assign end_addr_r[31:3] = ldst_dual_r ? end_addr_pre_r[31:3] : lsu_addr_r[31:3];       // This is for power saving
     319              : 
     320              :    rvdffe #(29) end_addr_hi_mff (.*, .din(end_addr_d[31:3]), .dout(end_addr_pre_m[31:3]), .en((lsu_pkt_d.valid & ldst_dual_d) | clk_override));
     321              :    rvdffe #(29) end_addr_hi_rff (.*, .din(end_addr_m[31:3]), .dout(end_addr_pre_r[31:3]), .en((lsu_pkt_m.valid & ldst_dual_m) | clk_override));
     322              : 
     323              :    rvdff #(3)  end_addr_lo_mff (.*, .din(end_addr_d[2:0]), .dout(end_addr_m[2:0]), .clk(lsu_c1_m_clk));
     324              :    rvdff #(3)  end_addr_lo_rff (.*, .din(end_addr_m[2:0]), .dout(end_addr_r[2:0]), .clk(lsu_c1_r_clk));
     325              : 
     326              :    rvdff #(1) addr_in_dccm_mff(.din(addr_in_dccm_d), .dout(addr_in_dccm_m), .clk(lsu_c1_m_clk), .*);
     327              :    rvdff #(1) addr_in_dccm_rff(.din(addr_in_dccm_m), .dout(addr_in_dccm_r), .clk(lsu_c1_r_clk), .*);
     328              : 
     329              :    rvdff #(1) addr_in_pic_mff(.din(addr_in_pic_d), .dout(addr_in_pic_m), .clk(lsu_c1_m_clk), .*);
     330              :    rvdff #(1) addr_in_pic_rff(.din(addr_in_pic_m), .dout(addr_in_pic_r), .clk(lsu_c1_r_clk), .*);
     331              : 
     332              :    rvdff #(1) addr_external_mff(.din(addr_external_d), .dout(addr_external_m), .clk(lsu_c1_m_clk), .*);
     333              :    rvdff #(1) addr_external_rff(.din(addr_external_m), .dout(addr_external_r), .clk(lsu_c1_r_clk), .*);
     334              : 
     335              :    rvdff #(1) access_fault_mff     (.din(access_fault_d),     .dout(access_fault_m),     .clk(lsu_c1_m_clk), .*);
     336              :    rvdff #(1) misaligned_fault_mff (.din(misaligned_fault_d), .dout(misaligned_fault_m), .clk(lsu_c1_m_clk), .*);
     337              :    rvdff #(4) exc_mscause_mff      (.din(exc_mscause_d[3:0]), .dout(exc_mscause_m[3:0]), .clk(lsu_c1_m_clk), .*);
     338              : 
     339              :    rvdff #(1) fir_dccm_access_error_mff    (.din(fir_dccm_access_error_d),    .dout(fir_dccm_access_error_m),    .clk(lsu_c1_m_clk), .*);
     340              :    rvdff #(1) fir_nondccm_access_error_mff (.din(fir_nondccm_access_error_d), .dout(fir_nondccm_access_error_m), .clk(lsu_c1_m_clk), .*);
     341              : 
     342              :    rvdffe #(32) bus_read_data_r_ff (.*, .din(bus_read_data_m[31:0]), .dout(bus_read_data_r[31:0]), .en(addr_external_m | clk_override));
     343              : 
     344              : endmodule