1

/*******************************************************************************

     2

     3

  Intel PRO/1000 Linux driver

     4

  Copyright(c) 1999 - 2009 Intel Corporation.

     5

     6

  This program is free software; you can redistribute it and/or modify it

     7

  under the terms and conditions of the GNU General Public License,

     8

  version 2, as published by the Free Software Foundation.

     9

    10

  This program is distributed in the hope it will be useful, but WITHOUT

    11

  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    12

  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for

    13

  more details.

    14

    15

  You should have received a copy of the GNU General Public License along with

    16

  this program; if not, write to the Free Software Foundation, Inc.,

    17

  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.

    18

    19

  The full GNU General Public License is included in this distribution in

    20

  the file called "COPYING".

    21

    22

  Contact Information:

    23

  Linux NICS <linux.nics@intel.com>

    24

  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>

    25

  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497

    26

    27

*******************************************************************************/

    28

    29

#include <linux/module.h>

    30

#include <linux/types.h>

    31

#include <linux/init.h>

    32

#include <linux/pci.h>

    33

#include <linux/vmalloc.h>

    34

#include <linux/pagemap.h>

    35

#include <linux/delay.h>

    36

#include <linux/netdevice.h>

    37

#include <linux/tcp.h>

    38

#include <linux/ipv6.h>

    39

#include <linux/slab.h>

    40

#include <net/checksum.h>

    41

#include <net/ip6_checksum.h>

    42

#include <linux/mii.h>

    43

#include <linux/ethtool.h>

    44

#include <linux/if_vlan.h>

    45

#include <linux/cpu.h>

    46

#include <linux/smp.h>

    47

#include <linux/pm_qos_params.h>

    48

#include <linux/aer.h>

    49

    50

#include "e1000.h"

    51

    52

#define DRV_VERSION "1.0.2-k2"

    53

char e1000e_driver_name[] = "e1000e";

    54

const char e1000e_driver_version[] = DRV_VERSION;

    55

    56

static const struct e1000_info *e1000_info_tbl[] = {

    57

	[board_82571]		= &e1000_82571_info,

    58

	[board_82572]		= &e1000_82572_info,

    59

	[board_82573]		= &e1000_82573_info,

    60

	[board_82574]		= &e1000_82574_info,

    61

	[board_82583]		= &e1000_82583_info,

    62

	[board_80003es2lan]	= &e1000_es2_info,

    63

	[board_ich8lan]		= &e1000_ich8_info,

    64

	[board_ich9lan]		= &e1000_ich9_info,

    65

	[board_ich10lan]	= &e1000_ich10_info,

    66

	[board_pchlan]		= &e1000_pch_info,

    67

};

    68

    69

/**

    70

 * e1000_desc_unused - calculate if we have unused descriptors

    71

 **/

    72

static int e1000_desc_unused(struct e1000_ring *ring)

    73

{

    74

	if (ring->next_to_clean > ring->next_to_use)

    75

		return ring->next_to_clean - ring->next_to_use - 1;

    76

    77

	return ring->count + ring->next_to_clean - ring->next_to_use - 1;

    78

}

    79

    80

/**

    81

 * e1000_receive_skb - helper function to handle Rx indications

    82

 * @adapter: board private structure

    83

 * @status: descriptor status field as written by hardware

    84

 * @vlan: descriptor vlan field as written by hardware (no le/be conversion)

    85

 * @skb: pointer to sk_buff to be indicated to stack

    86

 **/

    87

static void e1000_receive_skb(struct e1000_adapter *adapter,

    88

			      struct net_device *netdev,

    89

			      struct sk_buff *skb,

    90

			      u8 status, __le16 vlan)

    91

{

    92

	skb->protocol = eth_type_trans(skb, netdev);

    93

    94

	if (adapter->vlgrp && (status & E1000_RXD_STAT_VP))

    95

		vlan_gro_receive(&adapter->napi, adapter->vlgrp,

    96

				 le16_to_cpu(vlan), skb);

    97

	else

    98

		napi_gro_receive(&adapter->napi, skb);

    99

}

   100

   101

/**

   102

 * e1000_rx_checksum - Receive Checksum Offload for 82543

   103

 * @adapter:     board private structure

   104

 * @status_err:  receive descriptor status and error fields

   105

 * @csum:	receive descriptor csum field

   106

 * @sk_buff:     socket buffer with received data

   107

 **/

   108

static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err,

   109

			      u32 csum, struct sk_buff *skb)

   110

{

   111

	u16 status = (u16)status_err;

   112

	u8 errors = (u8)(status_err >> 24);

   113

	skb->ip_summed = CHECKSUM_NONE;

   114

   115

	/* Ignore Checksum bit is set */

   116

	if (status & E1000_RXD_STAT_IXSM)

   117

		return;

   118

	/* TCP/UDP checksum error bit is set */

   119

	if (errors & E1000_RXD_ERR_TCPE) {

   120

		/* let the stack verify checksum errors */

   121

		adapter->hw_csum_err++;

   122

		return;

   123

	}

   124

   125

	/* TCP/UDP Checksum has not been calculated */

   126

	if (!(status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)))

   127

		return;

   128

   129

	/* It must be a TCP or UDP packet with a valid checksum */

   130

	if (status & E1000_RXD_STAT_TCPCS) {

   131

		/* TCP checksum is good */

   132

		skb->ip_summed = CHECKSUM_UNNECESSARY;

   133

	} else {

   134

		/*

   135

		 * IP fragment with UDP payload

   136

		 * Hardware complements the payload checksum, so we undo it

   137

		 * and then put the value in host order for further stack use.

   138

		 */

   139

		__sum16 sum = (__force __sum16)htons(csum);

   140

		skb->csum = csum_unfold(~sum);

   141

		skb->ip_summed = CHECKSUM_COMPLETE;

   142

	}

   143

	adapter->hw_csum_good++;

   144

}

   145

   146

/**

   147

 * e1000_alloc_rx_buffers - Replace used receive buffers; legacy & extended

   148

 * @adapter: address of board private structure

   149

 **/

   150

static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter,

   151

				   int cleaned_count)

   152

{

   153

	struct net_device *netdev = adapter->netdev;

   154

	struct pci_dev *pdev = adapter->pdev;

   155

	struct e1000_ring *rx_ring = adapter->rx_ring;

   156

	struct e1000_rx_desc *rx_desc;

   157

	struct e1000_buffer *buffer_info;

   158

	struct sk_buff *skb;

   159

	unsigned int i;

   160

	unsigned int bufsz = adapter->rx_buffer_len;

   161

   162

	i = rx_ring->next_to_use;

   163

	buffer_info = &rx_ring->buffer_info[i];

   164

   165

	while (cleaned_count--) {

   166

		skb = buffer_info->skb;

   167

		if (skb) {

   168

			skb_trim(skb, 0);

   169

			goto map_skb;

   170

		}

   171

   172

		skb = netdev_alloc_skb_ip_align(netdev, bufsz);

   173

		if (!skb) {

   174

			/* Better luck next round */

   175

			adapter->alloc_rx_buff_failed++;

   176

			break;

   177

		}

   178

   179

		buffer_info->skb = skb;

   180

map_skb:

   181

		buffer_info->dma = pci_map_single(pdev, skb->data,

   182

						  adapter->rx_buffer_len,

   183

						  PCI_DMA_FROMDEVICE);

   184

		if (pci_dma_mapping_error(pdev, buffer_info->dma)) {

   185

			dev_err(&pdev->dev, "RX DMA map failed\n");

   186

			adapter->rx_dma_failed++;

   187

			break;

   188

		}

   189

   190

		rx_desc = E1000_RX_DESC(*rx_ring, i);

   191

		rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);

   192

   193

		i++;

   194

		if (i == rx_ring->count)

   195

			i = 0;

   196

		buffer_info = &rx_ring->buffer_info[i];

   197

	}

   198

   199

	if (rx_ring->next_to_use != i) {

   200

		rx_ring->next_to_use = i;

   201

		if (i-- == 0)

   202

			i = (rx_ring->count - 1);

   203

   204

		/*

   205

		 * Force memory writes to complete before letting h/w

   206

		 * know there are new descriptors to fetch.  (Only

   207

		 * applicable for weak-ordered memory model archs,

   208

		 * such as IA-64).

   209

		 */

   210

		wmb();

   211

		writel(i, adapter->hw.hw_addr + rx_ring->tail);

   212

	}

   213

}

   214

   215

/**

   216

 * e1000_alloc_rx_buffers_ps - Replace used receive buffers; packet split

   217

 * @adapter: address of board private structure

   218

 **/

   219

static void e1000_alloc_rx_buffers_ps(struct e1000_adapter *adapter,

   220

				      int cleaned_count)

   221

{

   222

	struct net_device *netdev = adapter->netdev;

   223

	struct pci_dev *pdev = adapter->pdev;

   224

	union e1000_rx_desc_packet_split *rx_desc;

   225

	struct e1000_ring *rx_ring = adapter->rx_ring;

   226

	struct e1000_buffer *buffer_info;

   227

	struct e1000_ps_page *ps_page;

   228

	struct sk_buff *skb;

   229

	unsigned int i, j;

   230

   231

	i = rx_ring->next_to_use;

   232

	buffer_info = &rx_ring->buffer_info[i];

   233

   234

	while (cleaned_count--) {

   235

		rx_desc = E1000_RX_DESC_PS(*rx_ring, i);

   236

   237

		for (j = 0; j < PS_PAGE_BUFFERS; j++) {

   238

			ps_page = &buffer_info->ps_pages[j];

   239

			if (j >= adapter->rx_ps_pages) {

   240

				/* all unused desc entries get hw null ptr */

   241

				rx_desc->read.buffer_addr[j+1] = ~cpu_to_le64(0);

   242

				continue;

   243

			}

   244

			if (!ps_page->page) {

   245

				ps_page->page = alloc_page(GFP_ATOMIC);

   246

				if (!ps_page->page) {

   247

					adapter->alloc_rx_buff_failed++;

   248

					goto no_buffers;

   249

				}

   250

				ps_page->dma = pci_map_page(pdev,

   251

						   ps_page->page,

   252

						   0, PAGE_SIZE,

   253

						   PCI_DMA_FROMDEVICE);

   254

				if (pci_dma_mapping_error(pdev, ps_page->dma)) {

   255

					dev_err(&adapter->pdev->dev,

   256

					  "RX DMA page map failed\n");

   257

					adapter->rx_dma_failed++;

   258

					goto no_buffers;

   259

				}

   260

			}

   261

			/*

   262

			 * Refresh the desc even if buffer_addrs

   263

			 * didn't change because each write-back

   264

			 * erases this info.

   265

			 */

   266

			rx_desc->read.buffer_addr[j+1] =

   267

			     cpu_to_le64(ps_page->dma);

   268

		}

   269

   270

		skb = netdev_alloc_skb_ip_align(netdev,

   271

						adapter->rx_ps_bsize0);

   272

   273

		if (!skb) {

   274

			adapter->alloc_rx_buff_failed++;

   275

			break;

   276

		}

   277

   278

		buffer_info->skb = skb;

   279

		buffer_info->dma = pci_map_single(pdev, skb->data,

   280

						  adapter->rx_ps_bsize0,

   281

						  PCI_DMA_FROMDEVICE);

   282

		if (pci_dma_mapping_error(pdev, buffer_info->dma)) {

   283

			dev_err(&pdev->dev, "RX DMA map failed\n");

   284

			adapter->rx_dma_failed++;

   285

			/* cleanup skb */

   286

			dev_kfree_skb_any(skb);

   287

			buffer_info->skb = NULL;

   288

			break;

   289

		}

   290

   291

		rx_desc->read.buffer_addr[0] = cpu_to_le64(buffer_info->dma);

   292

   293

		i++;

   294

		if (i == rx_ring->count)

   295

			i = 0;

   296

		buffer_info = &rx_ring->buffer_info[i];

   297

	}

   298

   299

no_buffers:

   300

	if (rx_ring->next_to_use != i) {

   301

		rx_ring->next_to_use = i;

   302

   303

		if (!(i--))

   304

			i = (rx_ring->count - 1);

   305

   306

		/*

   307

		 * Force memory writes to complete before letting h/w

   308

		 * know there are new descriptors to fetch.  (Only

   309

		 * applicable for weak-ordered memory model archs,

   310

		 * such as IA-64).

   311

		 */

   312

		wmb();

   313

		/*

   314

		 * Hardware increments by 16 bytes, but packet split

   315

		 * descriptors are 32 bytes...so we increment tail

   316

		 * twice as much.

   317

		 */

   318

		writel(i<<1, adapter->hw.hw_addr + rx_ring->tail);

   319

	}

   320

}

   321

   322

/**

   323

 * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers

   324

 * @adapter: address of board private structure

   325

 * @cleaned_count: number of buffers to allocate this pass

   326

 **/

   327

   328

static void e1000_alloc_jumbo_rx_buffers(struct e1000_adapter *adapter,

   329

                                         int cleaned_count)

   330

{

   331

	struct net_device *netdev = adapter->netdev;

   332

	struct pci_dev *pdev = adapter->pdev;

   333

	struct e1000_rx_desc *rx_desc;

   334

	struct e1000_ring *rx_ring = adapter->rx_ring;

   335

	struct e1000_buffer *buffer_info;

   336

	struct sk_buff *skb;

   337

	unsigned int i;

   338

	unsigned int bufsz = 256 - 16 /* for skb_reserve */;

   339

   340

	i = rx_ring->next_to_use;

   341

	buffer_info = &rx_ring->buffer_info[i];

   342

   343

	while (cleaned_count--) {

   344

		skb = buffer_info->skb;

   345

		if (skb) {

   346

			skb_trim(skb, 0);

   347

			goto check_page;

   348

		}

   349

   350

		skb = netdev_alloc_skb_ip_align(netdev, bufsz);

   351

		if (unlikely(!skb)) {

   352

			/* Better luck next round */

   353

			adapter->alloc_rx_buff_failed++;

   354

			break;

   355

		}

   356

   357

		buffer_info->skb = skb;

   358

check_page:

   359

		/* allocate a new page if necessary */

   360

		if (!buffer_info->page) {

   361

			buffer_info->page = alloc_page(GFP_ATOMIC);

   362

			if (unlikely(!buffer_info->page)) {

   363

				adapter->alloc_rx_buff_failed++;

   364

				break;

   365

			}

   366

		}

   367

   368

		if (!buffer_info->dma)

   369

			buffer_info->dma = pci_map_page(pdev,

   370

			                                buffer_info->page, 0,

   371

			                                PAGE_SIZE,

   372

			                                PCI_DMA_FROMDEVICE);

   373

   374

		rx_desc = E1000_RX_DESC(*rx_ring, i);

   375

		rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);

   376

   377

		if (unlikely(++i == rx_ring->count))

   378

			i = 0;

   379

		buffer_info = &rx_ring->buffer_info[i];

   380

	}

   381

   382

	if (likely(rx_ring->next_to_use != i)) {

   383

		rx_ring->next_to_use = i;

   384

		if (unlikely(i-- == 0))

   385

			i = (rx_ring->count - 1);

   386

   387

		/* Force memory writes to complete before letting h/w

   388

		 * know there are new descriptors to fetch.  (Only

   389

		 * applicable for weak-ordered memory model archs,

   390

		 * such as IA-64). */

   391

		wmb();

   392

		writel(i, adapter->hw.hw_addr + rx_ring->tail);

   393

	}

   394

}

   395

   396

/**

   397

 * e1000_clean_rx_irq - Send received data up the network stack; legacy

   398

 * @adapter: board private structure

   399

 *

   400

 * the return value indicates whether actual cleaning was done, there

   401

 * is no guarantee that everything was cleaned

   402

 **/

   403

static bool e1000_clean_rx_irq(struct e1000_adapter *adapter,

   404

			       int *work_done, int work_to_do)

   405

{

   406

	struct net_device *netdev = adapter->netdev;

   407

	struct pci_dev *pdev = adapter->pdev;

   408

	struct e1000_hw *hw = &adapter->hw;

   409

	struct e1000_ring *rx_ring = adapter->rx_ring;

   410

	struct e1000_rx_desc *rx_desc, *next_rxd;

   411

	struct e1000_buffer *buffer_info, *next_buffer;

   412

	u32 length;

   413

	unsigned int i;

   414

	int cleaned_count = 0;

   415

	bool cleaned = 0;

   416

	unsigned int total_rx_bytes = 0, total_rx_packets = 0;

   417

   418

	i = rx_ring->next_to_clean;

   419

	rx_desc = E1000_RX_DESC(*rx_ring, i);

   420

	buffer_info = &rx_ring->buffer_info[i];

   421

   422

	while (rx_desc->status & E1000_RXD_STAT_DD) {

   423

		struct sk_buff *skb;

   424

		u8 status;

   425

   426

		if (*work_done >= work_to_do)

   427

			break;

   428

		(*work_done)++;

   429

   430

		status = rx_desc->status;

   431

		skb = buffer_info->skb;

   432

		buffer_info->skb = NULL;

   433

   434

		prefetch(skb->data - NET_IP_ALIGN);

   435

   436

		i++;

   437

		if (i == rx_ring->count)

   438

			i = 0;

   439

		next_rxd = E1000_RX_DESC(*rx_ring, i);

   440

		prefetch(next_rxd);

   441

   442

		next_buffer = &rx_ring->buffer_info[i];

   443

   444

		cleaned = 1;

   445

		cleaned_count++;

   446

		pci_unmap_single(pdev,

   447

				 buffer_info->dma,

   448

				 adapter->rx_buffer_len,

   449

				 PCI_DMA_FROMDEVICE);

   450

		buffer_info->dma = 0;

   451

   452

		length = le16_to_cpu(rx_desc->length);

   453

   454

		/*

   455

		 * !EOP means multiple descriptors were used to store a single

   456

		 * packet, if that's the case we need to toss it.  In fact, we

   457

		 * need to toss every packet with the EOP bit clear and the

   458

		 * next frame that _does_ have the EOP bit set, as it is by

   459

		 * definition only a frame fragment

   460

		 */

   461

		if (unlikely(!(status & E1000_RXD_STAT_EOP)))

   462

			adapter->flags2 |= FLAG2_IS_DISCARDING;

   463

   464

		if (adapter->flags2 & FLAG2_IS_DISCARDING) {

   465

			/* All receives must fit into a single buffer */

   466

			e_dbg("Receive packet consumed multiple buffers\n");

   467

			/* recycle */

   468

			buffer_info->skb = skb;

   469

			if (status & E1000_RXD_STAT_EOP)

   470

				adapter->flags2 &= ~FLAG2_IS_DISCARDING;

   471

			goto next_desc;

   472

		}

   473

   474

		if (rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) {

   475

			/* recycle */

   476

			buffer_info->skb = skb;

   477

			goto next_desc;

   478

		}

   479

   480

		/* adjust length to remove Ethernet CRC */

   481

		if (!(adapter->flags2 & FLAG2_CRC_STRIPPING))

   482

			length -= 4;

   483

   484

		total_rx_bytes += length;

   485

		total_rx_packets++;

   486

   487

		/*

   488

		 * code added for copybreak, this should improve

   489

		 * performance for small packets with large amounts

   490

		 * of reassembly being done in the stack

   491

		 */

   492

		if (length < copybreak) {

   493

			struct sk_buff *new_skb =

   494

			    netdev_alloc_skb_ip_align(netdev, length);

   495

			if (new_skb) {

   496

				skb_copy_to_linear_data_offset(new_skb,

   497

							       -NET_IP_ALIGN,

   498

							       (skb->data -

   499

								NET_IP_ALIGN),

   500

							       (length +

   501

								NET_IP_ALIGN));

   502

				/* save the skb in buffer_info as good */

   503

				buffer_info->skb = skb;

   504

				skb = new_skb;

   505

			}

   506

			/* else just continue with the old one */

   507

		}

   508

		/* end copybreak code */

   509

		skb_put(skb, length);

   510

   511

		/* Receive Checksum Offload */

   512

		e1000_rx_checksum(adapter,

   513

				  (u32)(status) |

   514

				  ((u32)(rx_desc->errors) << 24),

   515

				  le16_to_cpu(rx_desc->csum), skb);

   516

   517

		e1000_receive_skb(adapter, netdev, skb,status,rx_desc->special);

   518

   519

next_desc:

   520

		rx_desc->status = 0;

   521

   522

		/* return some buffers to hardware, one at a time is too slow */

   523

		if (cleaned_count >= E1000_RX_BUFFER_WRITE) {

   524

			adapter->alloc_rx_buf(adapter, cleaned_count);

   525

			cleaned_count = 0;

   526

		}

   527

   528

		/* use prefetched values */

   529

		rx_desc = next_rxd;