1

/*******************************************************************************

     2

     3

  Intel PRO/1000 Linux driver

     4

  Copyright(c) 1999 - 2009 Intel Corporation.

     5

     6

  This program is free software; you can redistribute it and/or modify it

     7

  under the terms and conditions of the GNU General Public License,

     8

  version 2, as published by the Free Software Foundation.

     9

    10

  This program is distributed in the hope it will be useful, but WITHOUT

    11

  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    12

  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for

    13

  more details.

    14

    15

  You should have received a copy of the GNU General Public License along with

    16

  this program; if not, write to the Free Software Foundation, Inc.,

    17

  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.

    18

    19

  The full GNU General Public License is included in this distribution in

    20

  the file called "COPYING".

    21

    22

  Contact Information:

    23

  Linux NICS <linux.nics@intel.com>

    24

  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>

    25

  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497

    26

    27

*******************************************************************************/

    28

    29

#include <linux/module.h>

    30

#include <linux/types.h>

    31

#include <linux/init.h>

    32

#include <linux/pci.h>

    33

#include <linux/vmalloc.h>

    34

#include <linux/pagemap.h>

    35

#include <linux/delay.h>

    36

#include <linux/netdevice.h>

    37

#include <linux/tcp.h>

    38

#include <linux/ipv6.h>

    39

#include <net/checksum.h>

    40

#include <net/ip6_checksum.h>

    41

#include <linux/mii.h>

    42

#include <linux/ethtool.h>

    43

#include <linux/if_vlan.h>

    44

#include <linux/cpu.h>

    45

#include <linux/smp.h>

    46

#include <linux/pm_qos_params.h>

    47

#include <linux/aer.h>

    48

    49

#include "e1000.h"

    50

    51

#define DRV_VERSION "1.0.2-k2"

    52

char e1000e_driver_name[] = "e1000e";

    53

const char e1000e_driver_version[] = DRV_VERSION;

    54

    55

static const struct e1000_info *e1000_info_tbl[] = {

    56

	[board_82571]		= &e1000_82571_info,

    57

	[board_82572]		= &e1000_82572_info,

    58

	[board_82573]		= &e1000_82573_info,

    59

	[board_82574]		= &e1000_82574_info,

    60

	[board_82583]		= &e1000_82583_info,

    61

	[board_80003es2lan]	= &e1000_es2_info,

    62

	[board_ich8lan]		= &e1000_ich8_info,

    63

	[board_ich9lan]		= &e1000_ich9_info,

    64

	[board_ich10lan]	= &e1000_ich10_info,

    65

	[board_pchlan]		= &e1000_pch_info,

    66

};

    67

    68

/**

    69

 * e1000_desc_unused - calculate if we have unused descriptors

    70

 **/

    71

static int e1000_desc_unused(struct e1000_ring *ring)

    72

{

    73

	if (ring->next_to_clean > ring->next_to_use)

    74

		return ring->next_to_clean - ring->next_to_use - 1;

    75

    76

	return ring->count + ring->next_to_clean - ring->next_to_use - 1;

    77

}

    78

    79

/**

    80

 * e1000_receive_skb - helper function to handle Rx indications

    81

 * @adapter: board private structure

    82

 * @status: descriptor status field as written by hardware

    83

 * @vlan: descriptor vlan field as written by hardware (no le/be conversion)

    84

 * @skb: pointer to sk_buff to be indicated to stack

    85

 **/

    86

static void e1000_receive_skb(struct e1000_adapter *adapter,

    87

			      struct net_device *netdev,

    88

			      struct sk_buff *skb,

    89

			      u8 status, __le16 vlan)

    90

{

    91

	skb->protocol = eth_type_trans(skb, netdev);

    92

    93

	if (adapter->vlgrp && (status & E1000_RXD_STAT_VP))

    94

		vlan_gro_receive(&adapter->napi, adapter->vlgrp,

    95

				 le16_to_cpu(vlan), skb);

    96

	else

    97

		napi_gro_receive(&adapter->napi, skb);

    98

}

    99

   100

/**

   101

 * e1000_rx_checksum - Receive Checksum Offload for 82543

   102

 * @adapter:     board private structure

   103

 * @status_err:  receive descriptor status and error fields

   104

 * @csum:	receive descriptor csum field

   105

 * @sk_buff:     socket buffer with received data

   106

 **/

   107

static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err,

   108

			      u32 csum, struct sk_buff *skb)

   109

{

   110

	u16 status = (u16)status_err;

   111

	u8 errors = (u8)(status_err >> 24);

   112

	skb->ip_summed = CHECKSUM_NONE;

   113

   114

	/* Ignore Checksum bit is set */

   115

	if (status & E1000_RXD_STAT_IXSM)

   116

		return;

   117

	/* TCP/UDP checksum error bit is set */

   118

	if (errors & E1000_RXD_ERR_TCPE) {

   119

		/* let the stack verify checksum errors */

   120

		adapter->hw_csum_err++;

   121

		return;

   122

	}

   123

   124

	/* TCP/UDP Checksum has not been calculated */

   125

	if (!(status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)))

   126

		return;

   127

   128

	/* It must be a TCP or UDP packet with a valid checksum */

   129

	if (status & E1000_RXD_STAT_TCPCS) {

   130

		/* TCP checksum is good */

   131

		skb->ip_summed = CHECKSUM_UNNECESSARY;

   132

	} else {

   133

		/*

   134

		 * IP fragment with UDP payload

   135

		 * Hardware complements the payload checksum, so we undo it

   136

		 * and then put the value in host order for further stack use.

   137

		 */

   138

		__sum16 sum = (__force __sum16)htons(csum);

   139

		skb->csum = csum_unfold(~sum);

   140

		skb->ip_summed = CHECKSUM_COMPLETE;

   141

	}

   142

	adapter->hw_csum_good++;

   143

}

   144

   145

/**

   146

 * e1000_alloc_rx_buffers - Replace used receive buffers; legacy & extended

   147

 * @adapter: address of board private structure

   148

 **/

   149

static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter,

   150

				   int cleaned_count)

   151

{

   152

	struct net_device *netdev = adapter->netdev;

   153

	struct pci_dev *pdev = adapter->pdev;

   154

	struct e1000_ring *rx_ring = adapter->rx_ring;

   155

	struct e1000_rx_desc *rx_desc;

   156

	struct e1000_buffer *buffer_info;

   157

	struct sk_buff *skb;

   158

	unsigned int i;

   159

	unsigned int bufsz = adapter->rx_buffer_len;

   160

   161

	i = rx_ring->next_to_use;

   162

	buffer_info = &rx_ring->buffer_info[i];

   163

   164

	while (cleaned_count--) {

   165

		skb = buffer_info->skb;

   166

		if (skb) {

   167

			skb_trim(skb, 0);

   168

			goto map_skb;

   169

		}

   170

   171

		skb = netdev_alloc_skb_ip_align(netdev, bufsz);

   172

		if (!skb) {

   173

			/* Better luck next round */

   174

			adapter->alloc_rx_buff_failed++;

   175

			break;

   176

		}

   177

   178

		buffer_info->skb = skb;

   179

map_skb:

   180

		buffer_info->dma = pci_map_single(pdev, skb->data,

   181

						  adapter->rx_buffer_len,

   182

						  PCI_DMA_FROMDEVICE);

   183

		if (pci_dma_mapping_error(pdev, buffer_info->dma)) {

   184

			dev_err(&pdev->dev, "RX DMA map failed\n");

   185

			adapter->rx_dma_failed++;

   186

			break;

   187

		}

   188

   189

		rx_desc = E1000_RX_DESC(*rx_ring, i);

   190

		rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);

   191

   192

		i++;

   193

		if (i == rx_ring->count)

   194

			i = 0;

   195

		buffer_info = &rx_ring->buffer_info[i];

   196

	}

   197

   198

	if (rx_ring->next_to_use != i) {

   199

		rx_ring->next_to_use = i;

   200

		if (i-- == 0)

   201

			i = (rx_ring->count - 1);

   202

   203

		/*

   204

		 * Force memory writes to complete before letting h/w

   205

		 * know there are new descriptors to fetch.  (Only

   206

		 * applicable for weak-ordered memory model archs,

   207

		 * such as IA-64).

   208

		 */

   209

		wmb();

   210

		writel(i, adapter->hw.hw_addr + rx_ring->tail);

   211

	}

   212

}

   213

   214

/**

   215

 * e1000_alloc_rx_buffers_ps - Replace used receive buffers; packet split

   216

 * @adapter: address of board private structure

   217

 **/

   218

static void e1000_alloc_rx_buffers_ps(struct e1000_adapter *adapter,

   219

				      int cleaned_count)

   220

{

   221

	struct net_device *netdev = adapter->netdev;

   222

	struct pci_dev *pdev = adapter->pdev;

   223

	union e1000_rx_desc_packet_split *rx_desc;

   224

	struct e1000_ring *rx_ring = adapter->rx_ring;

   225

	struct e1000_buffer *buffer_info;

   226

	struct e1000_ps_page *ps_page;

   227

	struct sk_buff *skb;

   228

	unsigned int i, j;

   229

   230

	i = rx_ring->next_to_use;

   231

	buffer_info = &rx_ring->buffer_info[i];

   232

   233

	while (cleaned_count--) {

   234

		rx_desc = E1000_RX_DESC_PS(*rx_ring, i);

   235

   236

		for (j = 0; j < PS_PAGE_BUFFERS; j++) {

   237

			ps_page = &buffer_info->ps_pages[j];

   238

			if (j >= adapter->rx_ps_pages) {

   239

				/* all unused desc entries get hw null ptr */

   240

				rx_desc->read.buffer_addr[j+1] = ~cpu_to_le64(0);

   241

				continue;

   242

			}

   243

			if (!ps_page->page) {

   244

				ps_page->page = alloc_page(GFP_ATOMIC);

   245

				if (!ps_page->page) {

   246

					adapter->alloc_rx_buff_failed++;

   247

					goto no_buffers;

   248

				}

   249

				ps_page->dma = pci_map_page(pdev,

   250

						   ps_page->page,

   251

						   0, PAGE_SIZE,

   252

						   PCI_DMA_FROMDEVICE);

   253

				if (pci_dma_mapping_error(pdev, ps_page->dma)) {

   254

					dev_err(&adapter->pdev->dev,

   255

					  "RX DMA page map failed\n");

   256

					adapter->rx_dma_failed++;

   257

					goto no_buffers;

   258

				}

   259

			}

   260

			/*

   261

			 * Refresh the desc even if buffer_addrs

   262

			 * didn't change because each write-back

   263

			 * erases this info.

   264

			 */

   265

			rx_desc->read.buffer_addr[j+1] =

   266

			     cpu_to_le64(ps_page->dma);

   267

		}

   268

   269

		skb = netdev_alloc_skb_ip_align(netdev,

   270

						adapter->rx_ps_bsize0);

   271

   272

		if (!skb) {

   273

			adapter->alloc_rx_buff_failed++;

   274

			break;

   275

		}

   276

   277

		buffer_info->skb = skb;

   278

		buffer_info->dma = pci_map_single(pdev, skb->data,

   279

						  adapter->rx_ps_bsize0,

   280

						  PCI_DMA_FROMDEVICE);

   281

		if (pci_dma_mapping_error(pdev, buffer_info->dma)) {

   282

			dev_err(&pdev->dev, "RX DMA map failed\n");

   283

			adapter->rx_dma_failed++;

   284

			/* cleanup skb */

   285

			dev_kfree_skb_any(skb);

   286

			buffer_info->skb = NULL;

   287

			break;

   288

		}

   289

   290

		rx_desc->read.buffer_addr[0] = cpu_to_le64(buffer_info->dma);

   291

   292

		i++;

   293

		if (i == rx_ring->count)

   294

			i = 0;

   295

		buffer_info = &rx_ring->buffer_info[i];

   296

	}

   297

   298

no_buffers:

   299

	if (rx_ring->next_to_use != i) {

   300

		rx_ring->next_to_use = i;

   301

   302

		if (!(i--))

   303

			i = (rx_ring->count - 1);

   304

   305

		/*

   306

		 * Force memory writes to complete before letting h/w

   307

		 * know there are new descriptors to fetch.  (Only

   308

		 * applicable for weak-ordered memory model archs,

   309

		 * such as IA-64).

   310

		 */

   311

		wmb();

   312

		/*

   313

		 * Hardware increments by 16 bytes, but packet split

   314

		 * descriptors are 32 bytes...so we increment tail

   315

		 * twice as much.

   316

		 */

   317

		writel(i<<1, adapter->hw.hw_addr + rx_ring->tail);

   318

	}

   319

}

   320

   321

/**

   322

 * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers

   323

 * @adapter: address of board private structure

   324

 * @cleaned_count: number of buffers to allocate this pass

   325

 **/

   326

   327

static void e1000_alloc_jumbo_rx_buffers(struct e1000_adapter *adapter,

   328

                                         int cleaned_count)

   329

{

   330

	struct net_device *netdev = adapter->netdev;

   331

	struct pci_dev *pdev = adapter->pdev;

   332

	struct e1000_rx_desc *rx_desc;

   333

	struct e1000_ring *rx_ring = adapter->rx_ring;

   334

	struct e1000_buffer *buffer_info;

   335

	struct sk_buff *skb;

   336

	unsigned int i;

   337

	unsigned int bufsz = 256 - 16 /* for skb_reserve */;

   338

   339

	i = rx_ring->next_to_use;

   340

	buffer_info = &rx_ring->buffer_info[i];

   341

   342

	while (cleaned_count--) {

   343

		skb = buffer_info->skb;

   344

		if (skb) {

   345

			skb_trim(skb, 0);

   346

			goto check_page;

   347

		}

   348

   349

		skb = netdev_alloc_skb_ip_align(netdev, bufsz);

   350

		if (unlikely(!skb)) {

   351

			/* Better luck next round */

   352

			adapter->alloc_rx_buff_failed++;

   353

			break;

   354

		}

   355

   356

		buffer_info->skb = skb;

   357

check_page:

   358

		/* allocate a new page if necessary */

   359

		if (!buffer_info->page) {

   360

			buffer_info->page = alloc_page(GFP_ATOMIC);

   361

			if (unlikely(!buffer_info->page)) {

   362

				adapter->alloc_rx_buff_failed++;

   363

				break;

   364

			}

   365

		}

   366

   367

		if (!buffer_info->dma)

   368

			buffer_info->dma = pci_map_page(pdev,

   369

			                                buffer_info->page, 0,

   370

			                                PAGE_SIZE,

   371

			                                PCI_DMA_FROMDEVICE);

   372

   373

		rx_desc = E1000_RX_DESC(*rx_ring, i);

   374

		rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);

   375

   376

		if (unlikely(++i == rx_ring->count))

   377

			i = 0;

   378

		buffer_info = &rx_ring->buffer_info[i];

   379

	}

   380

   381

	if (likely(rx_ring->next_to_use != i)) {

   382

		rx_ring->next_to_use = i;

   383

		if (unlikely(i-- == 0))

   384

			i = (rx_ring->count - 1);

   385

   386

		/* Force memory writes to complete before letting h/w

   387

		 * know there are new descriptors to fetch.  (Only

   388

		 * applicable for weak-ordered memory model archs,

   389

		 * such as IA-64). */

   390

		wmb();

   391

		writel(i, adapter->hw.hw_addr + rx_ring->tail);

   392

	}

   393

}

   394

   395

/**

   396

 * e1000_clean_rx_irq - Send received data up the network stack; legacy

   397

 * @adapter: board private structure

   398

 *

   399

 * the return value indicates whether actual cleaning was done, there

   400

 * is no guarantee that everything was cleaned

   401

 **/

   402

static bool e1000_clean_rx_irq(struct e1000_adapter *adapter,

   403

			       int *work_done, int work_to_do)

   404

{

   405

	struct net_device *netdev = adapter->netdev;

   406

	struct pci_dev *pdev = adapter->pdev;

   407

	struct e1000_hw *hw = &adapter->hw;

   408

	struct e1000_ring *rx_ring = adapter->rx_ring;

   409

	struct e1000_rx_desc *rx_desc, *next_rxd;

   410

	struct e1000_buffer *buffer_info, *next_buffer;

   411

	u32 length;

   412

	unsigned int i;

   413

	int cleaned_count = 0;

   414

	bool cleaned = 0;

   415

	unsigned int total_rx_bytes = 0, total_rx_packets = 0;

   416

   417

	i = rx_ring->next_to_clean;

   418

	rx_desc = E1000_RX_DESC(*rx_ring, i);

   419

	buffer_info = &rx_ring->buffer_info[i];

   420

   421

	while (rx_desc->status & E1000_RXD_STAT_DD) {

   422

		struct sk_buff *skb;

   423

		u8 status;

   424

   425

		if (*work_done >= work_to_do)

   426

			break;

   427

		(*work_done)++;

   428

   429

		status = rx_desc->status;

   430

		skb = buffer_info->skb;

   431

		buffer_info->skb = NULL;

   432

   433

		prefetch(skb->data - NET_IP_ALIGN);

   434

   435

		i++;

   436

		if (i == rx_ring->count)

   437

			i = 0;

   438

		next_rxd = E1000_RX_DESC(*rx_ring, i);

   439

		prefetch(next_rxd);

   440

   441

		next_buffer = &rx_ring->buffer_info[i];

   442

   443

		cleaned = 1;

   444

		cleaned_count++;

   445

		pci_unmap_single(pdev,

   446

				 buffer_info->dma,

   447

				 adapter->rx_buffer_len,

   448

				 PCI_DMA_FROMDEVICE);

   449

		buffer_info->dma = 0;

   450

   451

		length = le16_to_cpu(rx_desc->length);

   452

   453

		/*

   454

		 * !EOP means multiple descriptors were used to store a single

   455

		 * packet, if that's the case we need to toss it.  In fact, we

   456

		 * need to toss every packet with the EOP bit clear and the

   457

		 * next frame that _does_ have the EOP bit set, as it is by

   458

		 * definition only a frame fragment

   459

		 */

   460

		if (unlikely(!(status & E1000_RXD_STAT_EOP)))

   461

			adapter->flags2 |= FLAG2_IS_DISCARDING;

   462

   463

		if (adapter->flags2 & FLAG2_IS_DISCARDING) {

   464

			/* All receives must fit into a single buffer */

   465

			e_dbg("Receive packet consumed multiple buffers\n");

   466

			/* recycle */

   467

			buffer_info->skb = skb;

   468

			if (status & E1000_RXD_STAT_EOP)

   469

				adapter->flags2 &= ~FLAG2_IS_DISCARDING;

   470

			goto next_desc;

   471

		}

   472

   473

		if (rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) {

   474

			/* recycle */

   475

			buffer_info->skb = skb;

   476

			goto next_desc;

   477

		}

   478

   479

		/* adjust length to remove Ethernet CRC */

   480

		if (!(adapter->flags2 & FLAG2_CRC_STRIPPING))

   481

			length -= 4;

   482

   483

		total_rx_bytes += length;

   484

		total_rx_packets++;

   485

   486

		/*

   487

		 * code added for copybreak, this should improve

   488

		 * performance for small packets with large amounts

   489

		 * of reassembly being done in the stack

   490

		 */

   491

		if (length < copybreak) {

   492

			struct sk_buff *new_skb =

   493

			    netdev_alloc_skb_ip_align(netdev, length);

   494

			if (new_skb) {

   495

				skb_copy_to_linear_data_offset(new_skb,

   496

							       -NET_IP_ALIGN,

   497

							       (skb->data -

   498

								NET_IP_ALIGN),

   499

							       (length +

   500

								NET_IP_ALIGN));

   501

				/* save the skb in buffer_info as good */

   502

				buffer_info->skb = skb;

   503

				skb = new_skb;

   504

			}

   505

			/* else just continue with the old one */

   506

		}

   507

		/* end copybreak code */

   508

		skb_put(skb, length);

   509

   510

		/* Receive Checksum Offload */

   511

		e1000_rx_checksum(adapter,

   512

				  (u32)(status) |

   513

				  ((u32)(rx_desc->errors) << 24),

   514

				  le16_to_cpu(rx_desc->csum), skb);

   515

   516

		e1000_receive_skb(adapter, netdev, skb,status,rx_desc->special);

   517

   518

next_desc:

   519

		rx_desc->status = 0;

   520

   521

		/* return some buffers to hardware, one at a time is too slow */

   522

		if (cleaned_count >= E1000_RX_BUFFER_WRITE) {

   523

			adapter->alloc_rx_buf(adapter, cleaned_count);

   524

			cleaned_count = 0;

   525

		}

   526

   527

		/* use prefetched values */

   528

		rx_desc = next_rxd;

   529

		buffer_info = next_buffer;